Skip to content

Commit 07bb4d6

Browse files
author
Emmanuel Merali
committed
Performance improvements for expire on rehash
Improves the performance of rehashing either with auto-rehash or with the _rehash method by pipelining the call to TYPE and TTL and using SETEX instead of SET and EXPIRE for string keys.
1 parent c7f5eeb commit 07bb4d6

File tree

1 file changed

+126
-83
lines changed

1 file changed

+126
-83
lines changed

‎redis_array_impl.c

Lines changed: 126 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -681,24 +681,55 @@ ra_rehash_scan_keys(zval *z_redis, char ***keys, int **key_lens TSRMLS_DC) {
681681
}
682682

683683
/* run TYPE to find the type */
684-
static long
685-
ra_get_key_type(zval *z_redis, const char *key, int key_len TSRMLS_DC) {
684+
static zend_bool
685+
ra_get_key_type(zval *z_redis, const char *key, int key_len, zval *z_from, long *res TSRMLS_DC) {
686686

687687
int i;
688688
zval z_fun_type, z_ret, *z_arg;
689+
zval **z_data;
690+
long success = 1;
691+
689692
MAKE_STD_ZVAL(z_arg);
693+
/* Pipelined */
694+
ra_index_multi(z_from, PIPELINE TSRMLS_CC);
690695

691696
/* prepare args */
692697
ZVAL_STRINGL(&z_fun_type, "TYPE", 4, 0);
693698
ZVAL_STRINGL(z_arg, key, key_len, 0);
699+
/* run TYPE */
700+
call_user_function(&redis_ce->function_table, &z_redis, &z_fun_type, &z_ret, 1, &z_arg TSRMLS_CC);
694701

702+
ZVAL_STRINGL(&z_fun_type, "TTL", 3, 0);
703+
ZVAL_STRINGL(z_arg, key, key_len, 0);
695704
/* run TYPE */
696705
call_user_function(&redis_ce->function_table, &z_redis, &z_fun_type, &z_ret, 1, &z_arg TSRMLS_CC);
697706

698707
/* cleanup */
699708
efree(z_arg);
700709

701-
return Z_LVAL(z_ret);
710+
/* Get the result from the pipeline. */
711+
ra_index_exec(z_from, &z_ret, 1 TSRMLS_CC);
712+
if(Z_TYPE(z_ret) == IS_ARRAY) {
713+
HashTable *retHash = Z_ARRVAL(z_ret);
714+
for(i = 0, zend_hash_internal_pointer_reset(retHash);
715+
zend_hash_has_more_elements(retHash) == SUCCESS;
716+
zend_hash_move_forward(retHash)) {
717+
718+
if(zend_hash_get_current_data(retHash, (void**)&z_data) == FAILURE) {
719+
success = 0;
720+
break;
721+
}
722+
if(Z_TYPE_PP(z_data) != IS_LONG) {
723+
success = 0;
724+
break;
725+
}
726+
/* Get the result - Might change in the future to handle doubles as well */
727+
res[i] = Z_LVAL_PP(z_data);
728+
i++;
729+
}
730+
}
731+
zval_dtor(&z_ret);
732+
return success;
702733
}
703734

704735
/* delete key from source server index during rehashing */
@@ -747,7 +778,29 @@ ra_del_key(const char *key, int key_len, zval *z_from TSRMLS_DC) {
747778
}
748779

749780
static zend_bool
750-
ra_move_zset(const char *key, int key_len, zval *z_from, zval *z_to TSRMLS_DC) {
781+
ra_expire_key(const char *key, int key_len, zval *z_to, long ttl TSRMLS_DC) {
782+
783+
zval z_fun_expire, z_ret, *z_args[2];
784+
785+
if (ttl > 0)
786+
{
787+
/* run EXPIRE on target */
788+
MAKE_STD_ZVAL(z_args[0]);
789+
MAKE_STD_ZVAL(z_args[1]);
790+
ZVAL_STRINGL(&z_fun_expire, "EXPIRE", 6, 0);
791+
ZVAL_STRINGL(z_args[0], key, key_len, 0);
792+
ZVAL_LONG(z_args[1], ttl);
793+
call_user_function(&redis_ce->function_table, &z_to, &z_fun_expire, &z_ret, 2, z_args TSRMLS_CC);
794+
/* cleanup */
795+
efree(z_args[0]);
796+
efree(z_args[1]);
797+
}
798+
799+
return 1;
800+
}
801+
802+
static zend_bool
803+
ra_move_zset(const char *key, int key_len, zval *z_from, zval *z_to, long ttl TSRMLS_DC) {
751804

752805
zval z_fun_zrange, z_fun_zadd, z_ret, *z_args[4], **z_zadd_args, **z_score_pp;
753806
int count;
@@ -820,6 +873,9 @@ ra_move_zset(const char *key, int key_len, zval *z_from, zval *z_to TSRMLS_DC) {
820873
ZVAL_STRINGL(z_zadd_args[0], key, key_len, 0);
821874
call_user_function(&redis_ce->function_table, &z_to, &z_fun_zadd, &z_ret, 1 + 2 * count, z_zadd_args TSRMLS_CC);
822875

876+
/* Expire if needed */
877+
ra_expire_key(key, key_len, z_to, ttl);
878+
823879
/* cleanup */
824880
for(i = 0; i < 1 + 2 * count; ++i) {
825881
efree(z_zadd_args[i]);
@@ -829,9 +885,9 @@ ra_move_zset(const char *key, int key_len, zval *z_from, zval *z_to TSRMLS_DC) {
829885
}
830886

831887
static zend_bool
832-
ra_move_string(const char *key, int key_len, zval *z_from, zval *z_to TSRMLS_DC) {
888+
ra_move_string(const char *key, int key_len, zval *z_from, zval *z_to, long ttl TSRMLS_DC) {
833889

834-
zval z_fun_get, z_fun_set, z_ret, *z_args[2];
890+
zval z_fun_get, z_fun_set, z_ret, *z_args[3];
835891

836892
/* run GET on source */
837893
MAKE_STD_ZVAL(z_args[0]);
@@ -847,21 +903,36 @@ ra_move_string(const char *key, int key_len, zval *z_from, zval *z_to TSRMLS_DC)
847903

848904
/* run SET on target */
849905
MAKE_STD_ZVAL(z_args[1]);
850-
ZVAL_STRINGL(&z_fun_set, "SET", 3, 0);
851-
ZVAL_STRINGL(z_args[0], key, key_len, 0);
852-
ZVAL_STRINGL(z_args[1], Z_STRVAL(z_ret), Z_STRLEN(z_ret), 1); /* copy z_ret to arg 1 */
853-
call_user_function(&redis_ce->function_table, &z_to, &z_fun_set, &z_ret, 2, z_args TSRMLS_CC);
906+
if (ttl > 0) {
907+
MAKE_STD_ZVAL(z_args[2]);
908+
ZVAL_STRINGL(&z_fun_set, "SETEX", 5, 0);
909+
ZVAL_STRINGL(z_args[0], key, key_len, 0);
910+
ZVAL_LONG(z_args[1], ttl);
911+
ZVAL_STRINGL(z_args[2], Z_STRVAL(z_ret), Z_STRLEN(z_ret), 1); /* copy z_ret to arg 1 */
912+
call_user_function(&redis_ce->function_table, &z_to, &z_fun_set, &z_ret, 3, z_args TSRMLS_CC);
913+
/* cleanup */
914+
efree(z_args[1]);
915+
zval_dtor(z_args[2]);
916+
efree(z_args[2]);
917+
}
918+
else {
919+
ZVAL_STRINGL(&z_fun_set, "SET", 3, 0);
920+
ZVAL_STRINGL(z_args[0], key, key_len, 0);
921+
ZVAL_STRINGL(z_args[1], Z_STRVAL(z_ret), Z_STRLEN(z_ret), 1); /* copy z_ret to arg 1 */
922+
call_user_function(&redis_ce->function_table, &z_to, &z_fun_set, &z_ret, 2, z_args TSRMLS_CC);
923+
/* cleanup */
924+
zval_dtor(z_args[1]);
925+
efree(z_args[1]);
926+
}
854927

855928
/* cleanup */
856929
efree(z_args[0]);
857-
zval_dtor(z_args[1]);
858-
efree(z_args[1]);
859930

860931
return 1;
861932
}
862933

863934
static zend_bool
864-
ra_move_hash(const char *key, int key_len, zval *z_from, zval *z_to TSRMLS_DC) {
935+
ra_move_hash(const char *key, int key_len, zval *z_from, zval *z_to, long ttl TSRMLS_DC) {
865936

866937
zval z_fun_hgetall, z_fun_hmset, z_ret, *z_args[2];
867938

@@ -883,6 +954,9 @@ ra_move_hash(const char *key, int key_len, zval *z_from, zval *z_to TSRMLS_DC) {
883954
z_args[1] = &z_ret; /* copy z_ret to arg 1 */
884955
call_user_function(&redis_ce->function_table, &z_to, &z_fun_hmset, &z_ret, 2, z_args TSRMLS_CC);
885956

957+
/* Expire if needed */
958+
ra_expire_key(key, key_len, z_to, ttl);
959+
886960
/* cleanup */
887961
efree(z_args[0]);
888962

@@ -892,7 +966,7 @@ ra_move_hash(const char *key, int key_len, zval *z_from, zval *z_to TSRMLS_DC) {
892966
static zend_bool
893967
ra_move_collection(const char *key, int key_len, zval *z_from, zval *z_to,
894968
int list_count, const char **cmd_list,
895-
int add_count, const char **cmd_add TSRMLS_DC) {
969+
int add_count, const char **cmd_add, long ttl TSRMLS_DC) {
896970

897971
zval z_fun_retrieve, z_fun_sadd, z_ret, **z_retrieve_args, **z_sadd_args, **z_data_pp;
898972
int count, i;
@@ -948,6 +1022,9 @@ ra_move_collection(const char *key, int key_len, zval *z_from, zval *z_to,
9481022
}
9491023
call_user_function(&redis_ce->function_table, &z_to, &z_fun_sadd, &z_ret, count+1, z_sadd_args TSRMLS_CC);
9501024

1025+
/* Expire if needed */
1026+
ra_expire_key(key, key_len, z_to, ttl);
1027+
9511028
/* cleanup */
9521029
efree(z_sadd_args[0]); /* no dtor at [0] */
9531030

@@ -961,93 +1038,59 @@ ra_move_collection(const char *key, int key_len, zval *z_from, zval *z_to,
9611038
}
9621039

9631040
static zend_bool
964-
ra_move_set(const char *key, int key_len, zval *z_from, zval *z_to TSRMLS_DC) {
1041+
ra_move_set(const char *key, int key_len, zval *z_from, zval *z_to, long ttl TSRMLS_DC) {
9651042

9661043
const char *cmd_list[] = {"SMEMBERS"};
9671044
const char *cmd_add[] = {"SADD"};
968-
return ra_move_collection(key, key_len, z_from, z_to, 1, cmd_list, 1, cmd_add TSRMLS_CC);
1045+
return ra_move_collection(key, key_len, z_from, z_to, 1, cmd_list, 1, cmd_add, ttl TSRMLS_CC);
9691046
}
9701047

9711048
static zend_bool
972-
ra_move_list(const char *key, int key_len, zval *z_from, zval *z_to TSRMLS_DC) {
1049+
ra_move_list(const char *key, int key_len, zval *z_from, zval *z_to, long ttl TSRMLS_DC) {
9731050

9741051
const char *cmd_list[] = {"LRANGE", "0", "-1"};
9751052
const char *cmd_add[] = {"RPUSH"};
976-
return ra_move_collection(key, key_len, z_from, z_to, 3, cmd_list, 1, cmd_add TSRMLS_CC);
977-
}
978-
979-
static zend_bool
980-
ra_expire_key(const char *key, int key_len, zval *z_from, zval *z_to TSRMLS_DC) {
981-
982-
zval z_fun_ttl, z_fun_expire, z_ret, *z_args[2];
983-
long ttl;
984-
985-
/* run TTL on source */
986-
MAKE_STD_ZVAL(z_args[0]);
987-
ZVAL_STRINGL(&z_fun_ttl, "TTL", 3, 0);
988-
ZVAL_STRINGL(z_args[0], key, key_len, 0);
989-
call_user_function(&redis_ce->function_table, &z_from, &z_fun_ttl, &z_ret, 1, z_args TSRMLS_CC);
990-
991-
if(Z_TYPE(z_ret) != IS_LONG) {
992-
efree(z_args[0]);
993-
return 0;
994-
}
995-
996-
ttl = Z_LVAL(z_ret);
997-
zval_dtor(&z_ret);
998-
if (ttl > 0)
999-
{
1000-
/* run EXPIRE on target */
1001-
MAKE_STD_ZVAL(z_args[1]);
1002-
ZVAL_STRINGL(&z_fun_expire, "EXPIRE", 6, 0);
1003-
ZVAL_STRINGL(z_args[0], key, key_len, 0);
1004-
ZVAL_LONG(z_args[1], ttl);
1005-
call_user_function(&redis_ce->function_table, &z_to, &z_fun_expire, &z_ret, 2, z_args TSRMLS_CC);
1006-
efree(z_args[1]);
1007-
}
1008-
/* cleanup */
1009-
efree(z_args[0]);
1010-
1011-
return 1;
1053+
return ra_move_collection(key, key_len, z_from, z_to, 3, cmd_list, 1, cmd_add, ttl TSRMLS_CC);
10121054
}
10131055

10141056
void
10151057
ra_move_key(const char *key, int key_len, zval *z_from, zval *z_to TSRMLS_DC) {
10161058

1017-
long type = ra_get_key_type(z_from, key, key_len TSRMLS_CC);
1059+
long res[2], type, ttl;
10181060
zend_bool success = 0;
1019-
1020-
/* open transaction on target server */
1021-
ra_index_multi(z_to, MULTI TSRMLS_CC);
1022-
1023-
switch(type) {
1024-
case REDIS_STRING:
1025-
success = ra_move_string(key, key_len, z_from, z_to TSRMLS_CC);
1026-
break;
1027-
1028-
case REDIS_SET:
1029-
success = ra_move_set(key, key_len, z_from, z_to TSRMLS_CC);
1030-
break;
1031-
1032-
case REDIS_LIST:
1033-
success = ra_move_list(key, key_len, z_from, z_to TSRMLS_CC);
1034-
break;
1035-
1036-
case REDIS_ZSET:
1037-
success = ra_move_zset(key, key_len, z_from, z_to TSRMLS_CC);
1038-
break;
1039-
1040-
case REDIS_HASH:
1041-
success = ra_move_hash(key, key_len, z_from, z_to TSRMLS_CC);
1042-
break;
1043-
1044-
default:
1045-
/* TODO: report? */
1046-
break;
1061+
if (ra_get_key_type(z_from, key, key_len, z_from, res TSRMLS_CC)) {
1062+
type = res[0];
1063+
ttl = res[1];
1064+
/* open transaction on target server */
1065+
ra_index_multi(z_to, MULTI TSRMLS_CC);
1066+
switch(type) {
1067+
case REDIS_STRING:
1068+
success = ra_move_string(key, key_len, z_from, z_to, ttl TSRMLS_CC);
1069+
break;
1070+
1071+
case REDIS_SET:
1072+
success = ra_move_set(key, key_len, z_from, z_to, ttl TSRMLS_CC);
1073+
break;
1074+
1075+
case REDIS_LIST:
1076+
success = ra_move_list(key, key_len, z_from, z_to, ttl TSRMLS_CC);
1077+
break;
1078+
1079+
case REDIS_ZSET:
1080+
success = ra_move_zset(key, key_len, z_from, z_to, ttl TSRMLS_CC);
1081+
break;
1082+
1083+
case REDIS_HASH:
1084+
success = ra_move_hash(key, key_len, z_from, z_to, ttl TSRMLS_CC);
1085+
break;
1086+
1087+
default:
1088+
/* TODO: report? */
1089+
break;
1090+
}
10471091
}
10481092

10491093
if(success) {
1050-
ra_expire_key(key, key_len, z_from, z_to TSRMLS_CC);
10511094
ra_del_key(key, key_len, z_from TSRMLS_CC);
10521095
ra_index_key(key, key_len, z_to TSRMLS_CC);
10531096
}

0 commit comments

Comments
 (0)