MMQ implementation for IQ4_KS_R4 and IQ5_KS_R4 (#493)

* MMQ for iq4_ks_r4

* MMQ for iq5_ks_r4

* Add forgotten file

* Another forgotten file

---------

Co-authored-by: Iwan Kawrakow <iwan.kawrakow@gmail.com>

Author: ikawrakow

ggml/src/ggml-cuda/common.cuh

@@ -599,6 +599,13 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ4_KS> {
     static constexpr int qi = QI4_XS;
 };
 
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ4_KS_R4> {
+    static constexpr int qk = QK_K;
+    static constexpr int qr = QR4_XS;
+    static constexpr int qi = QI4_XS;
+};
+
 template<>
 struct ggml_cuda_type_traits<GGML_TYPE_IQ4_KSS> {
     static constexpr int qk = QK_K;
@@ -620,6 +627,13 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ5_KS> {
     static constexpr int qi = QI5_XS;
 };
 
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ5_KS_R4> {
+    static constexpr int qk = QK_K;
+    static constexpr int qr = QR5_XS;
+    static constexpr int qi = QI5_XS;
+};
+
 template<>
 struct ggml_cuda_type_traits<GGML_TYPE_IQ6_K> {
     static constexpr int qk = QK_K;

ggml/src/ggml-cuda/iqk_mmvq.cu

@@ -36,21 +36,6 @@ struct ggml_cuda_type_traits<GGML_TYPE_IQ5_K_R4> {
     static constexpr int qi = QI5_XS;
 };
 
-template<>
-struct ggml_cuda_type_traits<GGML_TYPE_IQ4_KS_R4> {
-    static constexpr int qk = QK_K;
-    static constexpr int qr = QR4_XS;
-    static constexpr int qi = QI4_XS;
-};
-
-template<>
-struct ggml_cuda_type_traits<GGML_TYPE_IQ5_KS_R4> {
-    static constexpr int qk = QK_K;
-    static constexpr int qr = QR5_XS;
-    static constexpr int qi = QI5_XS;
-};
-
-
 //  Reminder:
 //    constexpr int qk  = ggml_cuda_type_traits<type>::qk;
 //    constexpr int qi  = ggml_cuda_type_traits<type>::qi;

ggml/src/ggml-cuda/mmq.cu

@@ -97,9 +97,15 @@ void ggml_cuda_op_mul_mat_q(
         case GGML_TYPE_IQ4_KS:
             mul_mat_q_case<GGML_TYPE_IQ4_KS>(ctx, args, stream);
             break;
+        case GGML_TYPE_IQ4_KS_R4:
+            mul_mat_q_case<GGML_TYPE_IQ4_KS_R4>(ctx, args, stream);
+            break;
         case GGML_TYPE_IQ5_KS:
             mul_mat_q_case<GGML_TYPE_IQ5_KS>(ctx, args, stream);
             break;
+        case GGML_TYPE_IQ5_KS_R4:
+            mul_mat_q_case<GGML_TYPE_IQ5_KS_R4>(ctx, args, stream);
+            break;
         case GGML_TYPE_IQ2_KS:
             mul_mat_q_case<GGML_TYPE_IQ2_KS>(ctx, args, stream);
             break;
@@ -157,7 +163,9 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11) {
         case GGML_TYPE_IQ4_XS:
         case GGML_TYPE_IQ4_NL:
         case GGML_TYPE_IQ4_KS:
+        case GGML_TYPE_IQ4_KS_R4:
         case GGML_TYPE_IQ5_KS:
+        case GGML_TYPE_IQ5_KS_R4:
         case GGML_TYPE_IQ2_KS:
         case GGML_TYPE_IQ2_K:
         case GGML_TYPE_IQ3_K: