Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

llama : add benchmark example #2626

Merged
merged 19 commits into from
Aug 18, 2023
Merged

llama : add benchmark example #2626

Changes from 1 commit
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
cfc7017
llama : add benchmark example
slaren Aug 15, 2023
7ec6158
add to examples CMakeLists.txt
slaren Aug 15, 2023
6597d61
fix msvc build
slaren Aug 15, 2023
6ab6971
add missing include
slaren Aug 15, 2023
52b94f4
add Bessel's correction to stdev calculation
slaren Aug 15, 2023
f2cf01d
improve markdown formatting
slaren Aug 16, 2023
f9bbc6f
add missing include
slaren Aug 16, 2023
19e9bea
print warning is NDEBUG is not defined
slaren Aug 16, 2023
3e3396e
remove n_prompt and n_gen from the matrix, use each value separately …
slaren Aug 16, 2023
5765f90
better checks for non-optimized builds
slaren Aug 16, 2023
89a70f7
llama.cpp : fix MEM_REQ_SCRATCH0 reusing the value of n_ctx of the fi…
slaren Aug 16, 2023
314a6b5
fix json formatting
slaren Aug 16, 2023
67362d9
add sql output
slaren Aug 17, 2023
cac7031
add basic cpu and gpu info (linx/cuda only)
slaren Aug 17, 2023
569dc6f
markdown: also show values that differ from the default
slaren Aug 17, 2023
94218e8
markdown: add build id
slaren Aug 17, 2023
9c38660
cleanup
slaren Aug 17, 2023
b6c81e2
improve formatting
slaren Aug 17, 2023
df87dd7
formatting
slaren Aug 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
better checks for non-optimized builds
  • Loading branch information
@slaren
slaren committed Aug 16, 2023
commit 5765f90f5873647e56cd6c30dd79c0f9b96af4de
18 changes: 13 additions & 5 deletions examples/llama-bench/llama-bench.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ struct cmd_params {
output_formats output_format;
};

static cmd_params cmd_params_defaults = {
static const cmd_params cmd_params_defaults = {
/* model */ {"models/7B/ggml-model-q4_0.bin"},
/* n_prompt */ {512},
/* n_gen */ {128},
Expand Down Expand Up @@ -688,9 +688,9 @@ struct markdown_printer : public printer {
value = backend_params::get_backend();
} else if (field == "test") {
char buf[128];
if (t.bparams.n_prompt > 0) {
if (t.bparams.n_prompt > 0 && t.bparams.n_gen == 0) {
snprintf(buf, sizeof(buf), "pp %d", t.bparams.n_prompt);
} else if (t.bparams.n_gen > 0) {
} else if (t.bparams.n_gen > 0 && t.bparams.n_prompt == 0) {
snprintf(buf, sizeof(buf), "tg %d", t.bparams.n_gen);
} else {
assert(false);
Expand Down Expand Up @@ -743,8 +743,16 @@ void llama_null_log_callback(enum llama_log_level level, const char * text, void
}

int main(int argc, char ** argv) {
#ifndef NDEBUG
fprintf(stderr, "warning: NDEBUG is not defined, performance may be affected\n");
#if !defined(NDEBUG)
fprintf(stderr, "warning: asserts enabled, performance may be affected\n");
#endif

#if (defined(_MSC_VER) && defined(_DEBUG)) || (!defined(_MSC_VER) && !defined(__OPTIMIZE__))
fprintf(stderr, "warning: debug build, performance may be affected\n");
#endif

#if defined(__SANITIZE_ADDRESS__) || defined(__SANITIZE_THREAD__)
fprintf(stderr, "warning: sanitizer enabled, performance may be affected\n");
#endif

cmd_params params = parse_cmd_params(argc, argv);
Expand Down