Skip to content

Commit 3b2836e

Browse files
SignalRTSignalRT
authored
Llava api (#563)
* Add llava_binaries, update all binaries to make the test * Llava API + LlavaTest Preliminary * First prototype of Load + Unit Test * Temporary run test con branch LlavaAPI * Disable Embed test to review the rest of the test * Restore Embedding test * Use BatchThread to eval image embeddings Test Threads default value to ensure it doesn´t produce problems. * Rename test file * Update action versions * Test only one method, no release embeddings * Revert "Test only one method, no release embeddings" This reverts commit 264e176. * Correct API call * Only test llava related functionality * Cuda and Cblast binaries * Restore build policy * Changes related with code review * Add SafeHandles * Set overwrite to upload-artifact@v4 * Revert to upload-artifact@v3 * revert to upload-artifact@v3
1 parent ce4de7d commit 3b2836e

14 files changed

+441
-17
lines changed

‎.github/workflows/compile.yml

Lines changed: 41 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,12 @@ jobs:
4848
cd build
4949
cmake .. ${{ env.COMMON_DEFINE }} ${{ matrix.defines }}
5050
cmake --build . --config Release -j ${env:NUMBER_OF_PROCESSORS}
51-
- uses: actions/upload-artifact@v4
51+
- uses: actions/upload-artifact@v3
5252
with:
5353
path: ./build/libllama.so
5454
name: llama-bin-linux-${{ matrix.build }}-x64.so
5555
- name: Upload Llava
56-
uses: actions/upload-artifact@v4
56+
uses: actions/upload-artifact@v3
5757
with:
5858
path: ./build/examples/llava/libllava_shared.so
5959
name: llava-bin-linux-${{ matrix.build }}-x64.so
@@ -89,13 +89,13 @@ jobs:
8989
cmake --build . --config Release -j ${env:NUMBER_OF_PROCESSORS}
9090
9191
- name: Upload artifacts
92-
uses: actions/upload-artifact@v4
92+
uses: actions/upload-artifact@v3
9393
with:
9494
path: .\build\bin\Release\llama.dll
9595
name: llama-bin-win-${{ matrix.build }}-x64.dll
9696

9797
- name: Upload Llava
98-
uses: actions/upload-artifact@v4
98+
uses: actions/upload-artifact@v3
9999
with:
100100
path: .\build\bin\Release\llava_shared.dll
101101
name: llava-bin-win-${{ matrix.build }}-x64.dll
@@ -169,20 +169,35 @@ jobs:
169169
ls -R
170170
- name: Upload artifacts (Windows)
171171
if: ${{ matrix.os == 'windows-latest' }}
172-
uses: actions/upload-artifact@v4
172+
uses: actions/upload-artifact@v3
173173
with:
174174
path: |
175175
.\build\bin\Release\llama.dll
176176
.\build\bin\Release\clblast.dll
177177
name: llama-bin-win-clblast-x64.dll
178+
- name: Upload llava artifacts (Windows)
179+
if: ${{ matrix.os == 'windows-latest' }}
180+
uses: actions/upload-artifact@v3
181+
with:
182+
path: |
183+
.\build\bin\Release\llava_shared.dll
184+
name: llava-bin-win-clblast-x64.dll
178185
- name: Upload artifacts (linux)
179186
if: ${{ matrix.os == 'ubuntu-22.04' }}
180-
uses: actions/upload-artifact@v4
187+
uses: actions/upload-artifact@v3
181188
with:
182189
path: |
183190
./build/libllama.so
184191
# ./build/libclblast.so
185192
name: llama-bin-linux-clblast-x64.so
193+
- name: Upload llava artifacts (linux)
194+
if: ${{ matrix.os == 'ubuntu-22.04' }}
195+
uses: actions/upload-artifact@v3
196+
with:
197+
path: |
198+
./build/examples/llava/libllava_shared.so
199+
name: llava-bin-linux-clblast-x64.so
200+
186201

187202
compile-cublas:
188203
name: Compile (cublas)
@@ -228,16 +243,29 @@ jobs:
228243
229244
- name: Upload artifacts (Windows)
230245
if: ${{ matrix.os == 'windows-latest' }}
231-
uses: actions/upload-artifact@v4
246+
uses: actions/upload-artifact@v3
232247
with:
233248
path: .\build\bin\Release\llama.dll
234249
name: llama-bin-win-cublas-cu${{ matrix.cuda }}-x64.dll
250+
- name: Upload llava artifacts (Windows)
251+
if: ${{ matrix.os == 'windows-latest' }}
252+
uses: actions/upload-artifact@v3
253+
with:
254+
path: .\build\bin\Release\llava_shared.dll
255+
name: llava-bin-win-cublas-cu${{ matrix.cuda }}-x64.dll
235256
- name: Upload artifacts (Linux)
236257
if: ${{ matrix.os == 'ubuntu-20.04' }}
237-
uses: actions/upload-artifact@v4
258+
uses: actions/upload-artifact@v3
238259
with:
239260
path: ./build/libllama.so
240261
name: llama-bin-linux-cublas-cu${{ matrix.cuda }}-x64.so
262+
- name: Upload llava artifacts (Linux)
263+
if: ${{ matrix.os == 'ubuntu-20.04' }}
264+
uses: actions/upload-artifact@v3
265+
with:
266+
path: ./build/examples/llava/libllava_shared.so
267+
name: llava-bin-linux-cublas-cu${{ matrix.cuda }}-x64.so
268+
241269

242270
compile-macos:
243271
name: Compile (MacOS)
@@ -268,18 +296,18 @@ jobs:
268296
cmake .. ${{ env.COMMON_DEFINE }} ${{ matrix.defines }}
269297
cmake --build . --config Release -j ${env:NUMBER_OF_PROCESSORS}
270298
- name: Upload artifacts
271-
uses: actions/upload-artifact@v4
299+
uses: actions/upload-artifact@v3
272300
with:
273301
path: ./build/libllama.dylib
274302
name: llama-bin-osx-${{ matrix.build }}.dylib
275303
- name: Upload Llava
276-
uses: actions/upload-artifact@v4
304+
uses: actions/upload-artifact@v3
277305
with:
278306
path: ./build/examples/llava/libllava_shared.dylib
279307
name: llava-bin-osx-${{ matrix.build }}.dylib
280308
- name: Upload Metal
281309
if: ${{ matrix.build != 'x64' }}
282-
uses: actions/upload-artifact@v4
310+
uses: actions/upload-artifact@v3
283311
with:
284312
path: ./build/bin/ggml-metal.metal
285313
name: ggml-metal.metal
@@ -347,11 +375,12 @@ jobs:
347375
cp artifacts/llama-bin-linux-clblast-x64.so/libllama.so deps/clblast/
348376
349377
- name: Upload artifacts
350-
uses: actions/upload-artifact@v4
378+
uses: actions/upload-artifact@v3
351379
with:
352380
path: deps/
353381
name: deps
354382

383+
355384
- name: Remove Artifacts
356385
uses: geekyeggo/delete-artifact@v2
357386
with:

‎.github/workflows/main.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,14 @@ jobs:
2828
os: windows-2019
2929
config: release
3030
steps:
31-
- uses: actions/checkout@v3
32-
- uses: actions/setup-dotnet@v3
31+
- uses: actions/checkout@v4
32+
- uses: actions/setup-dotnet@v4
3333
with:
3434
dotnet-version: |
3535
7.0.x
3636
8.0.x
3737
- name: Cache Packages
38-
uses: actions/cache@v3
38+
uses: actions/cache@v4
3939
with:
4040
key: "unit_test_models"
4141
path: LLama.Unittest/Models

‎LLama.Unittest/Constants.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,5 +3,8 @@
33
internal static class Constants
44
{
55
public static string ModelPath = "Models/llama-2-7b-chat.Q3_K_S.gguf";
6+
public static string LLavaModelPath = "Models/llava-v1.6-mistral-7b.Q3_K_XS.gguf";
7+
public static string LLavaMmpPath = "Models/mmproj-model-f16.gguf";
8+
public static string LLavaImage = "Models/extreme-ironing-taxi-610x427.jpg";
69
}
710
}

‎LLama.Unittest/LLama.Unittest.csproj

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,9 @@
2727
</ItemGroup>
2828

2929
<Target Name="DownloadContentFiles" BeforeTargets="Build">
30-
<DownloadFile SourceUrl="https://huggingface.co/TheBloke/Llama-2-7b-Chat-GGUF/resolve/main/llama-2-7b-chat.Q3_K_S.gguf" DestinationFolder="Models" DestinationFileName="llama-2-7b-chat.Q3_K_S.gguf" SkipUnchangedFiles="true">
31-
</DownloadFile>
30+
<DownloadFile SourceUrl="https://huggingface.co/TheBloke/Llama-2-7b-Chat-GGUF/resolve/main/llama-2-7b-chat.Q3_K_S.gguf" DestinationFolder="Models" DestinationFileName="llama-2-7b-chat.Q3_K_S.gguf" SkipUnchangedFiles="true"></DownloadFile>
31+
<DownloadFile SourceUrl="https://huggingface.co/cjpais/llava-1.6-mistral-7b-gguf/resolve/main/llava-v1.6-mistral-7b.Q3_K_XS.gguf" DestinationFolder="Models" DestinationFileName="llava-v1.6-mistral-7b.Q3_K_XS.gguf" SkipUnchangedFiles="true"></DownloadFile>
32+
<DownloadFile SourceUrl="https://huggingface.co/cjpais/llava-1.6-mistral-7b-gguf/resolve/main/mmproj-model-f16.gguf" DestinationFolder="Models" DestinationFileName="mmproj-model-f16.gguf" SkipUnchangedFiles="true"></DownloadFile>
3233
</Target>
3334

3435
<ItemGroup>
@@ -44,5 +45,14 @@
4445
<None Update="Models\llama-2-7b-chat.Q3_K_S.gguf">
4546
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
4647
</None>
48+
<None Update="Models\llava-v1.6-mistral-7b.Q3_K_XS.gguf">
49+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
50+
</None>
51+
<None Update="Models\mmproj-model-f16.gguf">
52+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
53+
</None>
54+
<None Update="Models\extreme-ironing-taxi-610x427.jpg">
55+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
56+
</None>
4757
</ItemGroup>
4858
</Project>

‎LLama.Unittest/LLamaEmbedderTests.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ public LLamaEmbedderTests(ITestOutputHelper testOutputHelper)
1414
_testOutputHelper = testOutputHelper;
1515
var @params = new ModelParams(Constants.ModelPath)
1616
{
17+
ContextSize = 4096,
18+
Threads = 5,
1719
EmbeddingMode = true,
1820
};
1921
using var weights = LLamaWeights.LoadFromFile(@params);
@@ -31,6 +33,7 @@ private static float Dot(float[] a, float[] b)
3133
return a.Zip(b, (x, y) => x * y).Sum();
3234
}
3335

36+
3437
[Fact]
3538
public async Task EmbedCompare()
3639
{

‎LLama.Unittest/LLavaWeightsTests.cs

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
using LLama.Common;
2+
using LLama.Native;
3+
4+
namespace LLama.Unittest
5+
{
6+
// Test the same things as llama model + image embedings
7+
//
8+
public sealed class LLavaWeightTests
9+
: IDisposable
10+
{
11+
private readonly LLamaWeights _llamaWeights;
12+
private readonly LLavaWeights _lLavaWeights;
13+
private readonly LLamaContext _context;
14+
15+
public LLavaWeightTests()
16+
{
17+
var @params = new ModelParams(Constants.ModelPath)
18+
{
19+
// Llava models requires big context
20+
ContextSize = 4096
21+
};
22+
_llamaWeights = LLamaWeights.LoadFromFile(@params);
23+
_lLavaWeights = LLavaWeights.LoadFromFile(Constants.LLavaMmpPath);
24+
25+
_context = _llamaWeights.CreateContext(@params);
26+
27+
}
28+
29+
public void Dispose()
30+
{
31+
_llamaWeights.Dispose();
32+
_lLavaWeights.Dispose();
33+
}
34+
35+
36+
37+
[Fact]
38+
public void EmbedImageAsFileName()
39+
{
40+
int n_past = 0;
41+
Assert.True( _lLavaWeights.EmbedImage( _context, Constants.LLavaImage, ref n_past ) );
42+
}
43+
44+
[Fact]
45+
public void EmbedImageAsBinary()
46+
{
47+
int n_past = 0;
48+
byte[] image = System.IO.File.ReadAllBytes(Constants.LLavaImage);
49+
Assert.True( _lLavaWeights.EmbedImage( _context, image, ref n_past ) );
50+
}
51+
52+
}
53+
}

‎LLama.Unittest/Models/extreme-ironing-taxi-610x427.jpg

90.4 KB
Loading

‎LLama/LLamaSharp.Runtime.targets

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,5 +67,51 @@
6767
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
6868
<Link>runtimes/osx-x64/native/libllama.dylib</Link>
6969
</None>
70+
71+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/llava_shared.dll">
72+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
73+
<Link>runtimes/win-x64/native/noavx/llava_shared.dll</Link>
74+
</None>
75+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx/llava_shared.dll">
76+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
77+
<Link>runtimes/win-x64/native/avx/llava_shared.dll</Link>
78+
</None>
79+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx2/llava_shared.dll">
80+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
81+
<Link>runtimes/win-x64/native/avx2/llava_shared.dll</Link>
82+
</None>
83+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx512/llava_shared.dll">
84+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
85+
<Link>runtimes/win-x64/native/avx512/llava_shared.dll</Link>
86+
</None>
87+
88+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/libllava_shared.so">
89+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
90+
<Link>runtimes/linux-x64/native/noavx/libllava_shared.so</Link>
91+
</None>
92+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx/libllava_shared.so">
93+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
94+
<Link>runtimes/linux-x64/native/avx/libllava_shared.so</Link>
95+
</None>
96+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx2/libllava_shared.so">
97+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
98+
<Link>runtimes/linux-x64/native/avx2/libllava_shared.so</Link>
99+
</None>
100+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx512/libllava_shared.so">
101+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
102+
<Link>runtimes/linux-x64/native/avx512/libllava_shared.so</Link>
103+
</None>
104+
105+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-arm64/libllava_shared.dylib">
106+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
107+
<Link>runtimes/osx-arm64/native/libllava_shared.dylib</Link>
108+
</None>
109+
110+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-x64/libllava_shared.dylib">
111+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
112+
<Link>runtimes/osx-x64/native/libllava_shared.dylib</Link>
113+
</None>
114+
115+
70116
</ItemGroup>
71117
</Project>

‎LLama/LLavaWeights.cs

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
2+
using System;
3+
using LLama.Native;
4+
5+
namespace LLama;
6+
7+
public sealed class LLavaWeights : IDisposable
8+
{
9+
public SafeLlavaModelHandle NativeHandle { get; }
10+
11+
internal LLavaWeights(SafeLlavaModelHandle weights)
12+
{
13+
NativeHandle = weights;
14+
}
15+
16+
public static LLavaWeights LoadFromFile(string mmProject)
17+
{
18+
var weights = SafeLlavaModelHandle.LoadFromFile(mmProject, 1);
19+
return new LLavaWeights(weights);
20+
}
21+
22+
/// <summary>
23+
/// Embed the image from file into llama context
24+
/// </summary>
25+
/// <param name="ctxLlama"></param>
26+
/// <param name="Image"></param>
27+
/// <param name="n_past"></param>
28+
/// <returns></returns>
29+
public bool EmbedImage(LLamaContext ctxLlama, string Image, ref int n_past )
30+
{
31+
return NativeHandle.EmbedImage(ctxLlama, Image, ref n_past );
32+
}
33+
34+
/// <summary>
35+
/// Embed the image from binary into llama context.
36+
/// </summary>
37+
/// <param name="ctxLlama"></param>
38+
/// <param name="Image"></param>
39+
/// <param name="n_past"></param>
40+
/// <returns></returns>
41+
public bool EmbedImage(LLamaContext ctxLlama, Byte[] Image, ref int n_past )
42+
{
43+
return NativeHandle.EmbedImage(ctxLlama, Image, ref n_past );
44+
}
45+
46+
public void Dispose()
47+
{
48+
NativeHandle.Dispose();
49+
}
50+
51+
}

‎LLama/Native/LLavaImageEmbed.cs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
using System.Runtime.InteropServices;
2+
3+
namespace LLama.Native;
4+
5+
/// <summary>
6+
/// LLaVa Image embeddings
7+
/// </summary>
8+
[StructLayout(LayoutKind.Sequential)]
9+
unsafe public struct LLavaImageEmbed
10+
{
11+
public float* embed;
12+
public int n_image_pos;
13+
}

0 commit comments

Comments
 (0)