Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ggml : remove bit shuffling #1305

Closed
wants to merge 20 commits into from
Closed

ggml : remove bit shuffling #1305

Changes from 1 commit
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
a546dc6
ggml : remove Q4_0 bit shufling (ARM NEON)
ggerganov May 3, 2023
edb6c8b
ggml : remove Q4_1 bit shuffling (ARM NEON + reference)
ggerganov May 4, 2023
086cfea
ggml : nibbles_from_floats() + bytes_from_nibbles() (ARM NEON)
ggerganov May 4, 2023
a6a1d96
ggml : remove Q4_2 bit shuffling (WIP, BROKEN)
ggerganov May 4, 2023
796f8ae
ggml : remove Q5_0 bit shuffling (ARM NEON)
ggerganov May 4, 2023
39bb8e7
ggml : 2x faster scalar implementations
ggerganov May 4, 2023
c7af904
ggml : remove Q5_1 bit shuffling (ARM NEON + scalar)
ggerganov May 5, 2023
ba953d6
ggml : simplify scalar dot
ggerganov May 5, 2023
4991499
ggml : remove WASM SIMD bit shuffling + remove vzip for ARM 32-bit
ggerganov May 5, 2023
c216656
ggml : fix Q4_1 quantization
ggerganov May 7, 2023
b47bd28
ggml : update cuBLAS + normalize variable names
ggerganov May 7, 2023
7cdc08a
ggml : remove Q4_2 mode
ggerganov May 7, 2023
60f62bb
ggml : minor formatting
ggerganov May 7, 2023
8fbf777
ggml : fix Q5_0 quantization
ggerganov May 7, 2023
d155f0f
scripts : add script for measuring the time per token
ggerganov May 8, 2023
948d124
AVX implementations (#1370)
sw May 8, 2023
0e48eb6
ggml : uniform 5th bit extraction
ggerganov May 8, 2023
4201fa5
llama : produce error upon loading old model files
ggerganov May 9, 2023
ffd76e1
llama : fix model magic/version write
ggerganov May 9, 2023
e116eb6
ggml : speed-up Q5_0 + Q5_1 at 4 threads
ggerganov May 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
ggml : nibbles_from_floats() + bytes_from_nibbles() (ARM NEON)
  • Loading branch information
@ggerganov
ggerganov committed May 8, 2023
commit 086cfea11fc59762696ac29052f77437e1dda24e
131 changes: 60 additions & 71 deletions ggml.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -615,6 +615,50 @@ static inline __m128i packNibbles( __m128i bytes1, __m128i bytes2 )

#if __ARM_NEON

static inline const uint8_t * bytes_from_nibbles_64(const int qk, const uint8_t * qs, uint64_t * qd) {
memcpy(qd, qs, qk/2);

for (int l = 0; l < qk/16; ++l) {
qd[l + qk/16] = (qd[l] & 0xF0F0F0F0F0F0F0F0ULL) >> 4;
qd[l + 0 ] = (qd[l] & 0x0F0F0F0F0F0F0F0FULL) >> 0;
}

return (const uint8_t *) qd;
}

// pack first half of weights into low nibbles and second half into high nibbles
// use one scaling factor
static inline void nibbles_from_floats_64_0(const int qk, const float * x, float id, uint8_t * qs, uint64_t * qd) {
for (int l = 0; l < qk/2; ++l) {
const float v0 = x[0 + l]*id;
const float v1 = x[qk/2 + l]*id;

const uint64_t vi0 = MIN(15, (int8_t)(v0 + 8.5f));
const uint64_t vi1 = MIN(15, (int8_t)(v1 + 8.5f));

qd[l/8] |= vi0 << (8*(l & 7));
qd[l/8] |= vi1 << (8*(l & 7) + 4);
}

memcpy(qs, qd, qk/2);
}

// use offset and scaling factor
static inline void nibbles_from_floats_64_1(const int qk, const float * x, float id, float min, uint8_t * qs, uint64_t * qd) {
for (int l = 0; l < qk/2; ++l) {
const float v0 = (x[0 + l] - min)*id;
const float v1 = (x[qk/2 + l] - min)*id;

const uint64_t vi0 = MIN(15, (int8_t)(v0 + 0.5f));
const uint64_t vi1 = MIN(15, (int8_t)(v1 + 0.5f));

qd[l/8] |= vi0 << (8*(l & 7));
qd[l/8] |= vi1 << (8*(l & 7) + 4);
}

memcpy(qs, qd, qk/2);
}

#if !defined(__aarch64__)

inline static uint16_t vaddvq_u8(uint8x16_t v) {
Expand Down Expand Up @@ -863,19 +907,7 @@ static void quantize_row_q4_0_reference(const float * restrict x, block_q4_0 * r

uint64_t qs[QK4_0 / 16] = {0};

// pack first half of weights into low nibbles and second half into high nibbles
for (int l = 0; l < qk/2; ++l) {
const float v0 = x[i*qk + 0 + l]*id;
const float v1 = x[i*qk + qk/2 + l]*id;

const uint64_t vi0 = MIN(15, (int8_t)(v0 + 8.5f));
const uint64_t vi1 = MIN(15, (int8_t)(v1 + 8.5f));

qs[l/8] |= vi0 << (8*(l & 7));
qs[l/8] |= vi1 << (8*(l & 7) + 4);
}

memcpy(y[i].qs, qs, sizeof(qs));
nibbles_from_floats_64_0(qk, x + i*qk, id, y[i].qs, qs);
}
}

Expand Down Expand Up @@ -910,19 +942,7 @@ static void quantize_row_q4_1_reference(const float * restrict x, block_q4_1 * r

uint64_t qs[QK4_1 / 16] = {0};

// pack first half of weights into low nibbles and second half into high nibbles
for (int l = 0; l < qk/2; ++l) {
const float v0 = (x[i*qk + 0 + l] - min)*id;
const float v1 = (x[i*qk + qk/2 + l] - min)*id;

const uint64_t vi0 = MIN(15, (int8_t)(v0 + 0.5f));
const uint64_t vi1 = MIN(15, (int8_t)(v1 + 0.5f));

qs[l/8] |= vi0 << (8*(l & 7));
qs[l/8] |= vi1 << (8*(l & 7) + 4);
}

memcpy(y[i].qs, qs, sizeof(qs));
nibbles_from_floats_64_1(qk, x + i*qk, id, min, y[i].qs, qs);
}
}

Expand Down Expand Up @@ -1435,20 +1455,12 @@ static void dequantize_row_q4_0(const block_q4_0 * restrict x, float * restrict

const int nb = k / qk;

uint64_t qs[QK4_0 / 8];

for (int i = 0; i < nb; i++) {
const float d = x[i].d;

// unpack nibbles into bytes
uint64_t qs[QK4_0 / 8] = {0};

memcpy(qs + 0, x[i].qs, sizeof(x[i].qs));

for (int l = 0; l < qk / 16; ++l) {
qs[l + qk/16] = (qs[l] & 0xF0F0F0F0F0F0F0F0ULL) >> 4;
qs[l + 0 ] = (qs[l] & 0x0F0F0F0F0F0F0F0FULL) >> 0;
}

const uint8_t * restrict qsp = (const uint8_t * restrict) qs;
const uint8_t * qsp = bytes_from_nibbles_64(qk, x[i].qs, qs);

for (int l = 0; l < qk; ++l) {
y[i*qk + l] = (qsp[l] - 8)*d;
Expand All @@ -1464,21 +1476,13 @@ static void dequantize_row_q4_1(const block_q4_1 * restrict x, float * restrict

const int nb = k / qk;

uint64_t qs[QK4_0 / 8];

for (int i = 0; i < nb; i++) {
const float d = x[i].d;
const float m = x[i].m;

// unpack nibbles into bytes
uint64_t qs[QK4_0 / 8] = {0};

memcpy(qs + 0, x[i].qs, sizeof(x[i].qs));

for (int l = 0; l < qk / 16; ++l) {
qs[l + qk/16] = (qs[l] & 0xF0F0F0F0F0F0F0F0ULL) >> 4;
qs[l + 0 ] = (qs[l] & 0x0F0F0F0F0F0F0F0FULL) >> 0;
}

const uint8_t * restrict qsp = (const uint8_t * restrict) qs;
const uint8_t * qsp = bytes_from_nibbles_64(qk, x[i].qs, qs);

for (int l = 0; l < qk; ++l) {
y[i*qk + l] = qsp[l]*d + m;
Expand Down Expand Up @@ -2410,19 +2414,12 @@ static void ggml_vec_dot_q4_0_q8_0(const int n, float * restrict s, const void *
// scalar
float sumf = 0.0;

uint64_t qs[QK8_0 / 8];

for (int i = 0; i < nb; i++) {
// unpack nibbles into bytes
uint64_t qs[QK8_0 / 8] = {0};

memcpy(qs + 0, x[i].qs, sizeof(x[i].qs));

for (int l = 0; l < qk / 16; ++l) {
qs[l + qk/16] = (qs[l] & 0xF0F0F0F0F0F0F0F0ULL) >> 4;
qs[l + 0 ] = (qs[l] & 0x0F0F0F0F0F0F0F0FULL) >> 0;
}

const uint8_t * restrict px = (const uint8_t * restrict) qs;
const int8_t * restrict py = y[i].qs;
const uint8_t * px = bytes_from_nibbles_64(qk, x[i].qs, qs);
const int8_t * py = y[i].qs;

int sumi = 0;

Expand Down Expand Up @@ -2542,19 +2539,11 @@ static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, const void *
// scalar
float sumf = 0.0;

for (int i = 0; i < nb; i++) {
// unpack nibbles into bytes
uint64_t qs[QK8_1 / 8] = {0};

memcpy(qs + 0, x[i].qs, sizeof(x[i].qs));
uint64_t qs[QK8_1 / 8];

for (int l = 0; l < qk / 16; ++l) {
qs[l + qk/16] = (qs[l] & 0xF0F0F0F0F0F0F0F0ULL) >> 4;
qs[l + 0 ] = (qs[l] & 0x0F0F0F0F0F0F0F0FULL) >> 0;
}

const uint8_t * restrict px = (const uint8_t * restrict) qs;
const int8_t * restrict py = y[i].qs;
for (int i = 0; i < nb; i++) {
const uint8_t * px = bytes_from_nibbles_64(qk, x[i].qs, qs);
const int8_t * py = y[i].qs;

int sumi = 0;

Expand Down