updated develop to be in sync with antirez/5.0.9

00-RELEASENOTES

@@ -11,6 +11,27 @@ CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP.
 SECURITY: There are security fixes in the release.
 --------------------------------------------------------------------------------
 
+================================================================================
+Redis 5.0.9     Released Thu Apr 17 12:41:00 CET 2020
+================================================================================
+
+Upgrade urgency:CRITICAL if you use Streams with AOF ore replicas.
+                Otherwise the upgrade urgency is LOW.
+
+This release has a speed improvement and a critical fix:
+
+    * FIX: XREADGROUP when fetching data in a blocking way, would not
+           emit the XCLAIM in the AOF file and to replicas. This means
+           that the last ID is not updated, and that restarting the server
+           will have the effect of reprocessing some entries.
+    * NEW: Clients blocked on the same key are now unblocked on
+           O(1) time. Backported from Redis 6.
+
+Commits:
+
+    1fc8ef81a Fix XCLAIM propagation in AOF/replicas for blocking XREADGROUP.
+    a5e24eabc Speedup: unblock clients on keys in O(1).
+
 ================================================================================
 Redis 5.0.8     Released Thu Mar 12 16:05:41 CET 2020
 ================================================================================

README.md

@@ -1,6 +1,6 @@
 # Redis 5.0.x for Windows
 
-Branch **develop** is still under development and at the moment provides an unstable version of Redis 5.0.8. See below for information on latest release from 4.0.x branch.
+Branch **develop** is still under development and at the moment provides an unstable version of Redis 5.0.9. See below for information on latest release from 4.0.x branch.
 
 ## Redis 4.0.14 for Windows
 

msvs/Build.bat

@@ -0,0 +1,18 @@
+@echo off
+
+set CONFIGURATION=%1
+if not defined CONFIGURATION (
+	set CONFIGURATION=Release
+)
+
+echo Building with configuration = %CONFIGURATION% in 5 seconds
+rem wait for 5 seconds... (https://stackoverflow.com/a/735603)
+ping -n 6 127.0.0.1 > nul
+
+msbuild RedisServer.sln -t:Rebuild -p:Configuration=%CONFIGURATION%;Platform=x64;Machine=x64
+msbuild RedisServer.sln -t:RedisCli -p:Configuration=%CONFIGURATION%;Platform=x64;Machine=x64
+msbuild RedisServer.sln -t:RedisBenchmark -p:Configuration=%CONFIGURATION%;Platform=x64;Machine=x64
+
+rem cd msi
+rem msbuild RedisMsi.sln -t:Rebuild -p:Configuration=%CONFIGURATION%;Platform=x64
+rem cd ..

msvs/RedisForWindows.rc

@@ -50,8 +50,8 @@ END
 //
 
 VS_VERSION_INFO VERSIONINFO
- FILEVERSION 5,0,8
- PRODUCTVERSION 5,0,8
+ FILEVERSION 5,0,9
+ PRODUCTVERSION 5,0,9
  FILEFLAGSMASK 0x3fL
 #ifdef _DEBUG
  FILEFLAGS 0x1L
@@ -68,10 +68,10 @@ BEGIN
         BEGIN
             VALUE "CompanyName", "Poradowski.com Tomasz Poradowski"
             VALUE "FileDescription", "Redis for Windows, based on MS OpenTech port."
-            VALUE "FileVersion", "5.0.8"
+            VALUE "FileVersion", "5.0.9"
             VALUE "LegalCopyright", "Copyright (C) 2006-2018 Salvatore Sanfilippo, Microsoft Open Technologies Inc., Tomasz Poradowski"
             VALUE "ProductName", "Redis for Windows"
-            VALUE "ProductVersion", "5.0.8"
+            VALUE "ProductVersion", "5.0.9"
         END
     END
     BLOCK "VarFileInfo"

msvs/msi/RedisMsi/Product.wxs

@@ -21,7 +21,7 @@
     <Product Id="*"
              Name="Redis on Windows"
              Language="1033"
-             Version="5.0.8"
+             Version="5.0.9"
              Manufacturer="Poradowski.com"
              UpgradeCode="{05410198-7212-4FC4-B7C8-AFEFC3DA0FBC}">
         <Package InstallerVersion="200"

msvs/setups/chocolatey/Redis.nuspec

@@ -3,7 +3,7 @@
   <metadata>
     <id>redis-64</id>
     <title>Redis 64-bit</title>
-    <version>5.0.8</version>
+    <version>5.0.9</version>
     <authors>Alexis Campailla, Enrico Giordani, Jonathan Pickett</authors>
     <owners>Microsoft Open Technologies, Inc.</owners>
     <description>A porting of Redis on Windows 64-bit.  

msvs/setups/nuget/Redis.nuspec

@@ -3,7 +3,7 @@
   <metadata>
     <id>redis-64</id>
     <title>Redis 64-bit</title>
-    <version>5.0.8</version>
+    <version>5.0.9</version>
     <authors>Alexis Campailla, Enrico Giordani, Jonathan Pickett</authors>
     <owners>Microsoft Open Technologies, Inc.</owners>
     <description>A porting of Redis on Windows 64-bit.  

src/adlist.c

@@ -330,12 +330,11 @@ listNode *listIndex(list *list, PORT_LONG index) {
 }
 
 /* Rotate the list removing the tail node and inserting it to the head. */
-void listRotate(list *list) {
-    listNode *tail = list->tail;
-
+void listRotateTailToHead(list *list) {
     if (listLength(list) <= 1) return;
 
     /* Detach current tail */
+    listNode *tail = list->tail;
     list->tail = tail->prev;
     list->tail->next = NULL;
     /* Move it as head */
@@ -345,6 +344,21 @@ void listRotate(list *list) {
     list->head = tail;
 }
 
+/* Rotate the list removing the head node and inserting it to the tail. */
+void listRotateHeadToTail(list *list) {
+    if (listLength(list) <= 1) return;
+
+    listNode *head = list->head;
+    /* Detach current head */
+    list->head = head->next;
+    list->head->prev = NULL;
+    /* Move it as tail */
+    list->tail->next = head;
+    head->next = NULL;
+    head->prev = list->tail;
+    list->tail = head;
+}
+
 /* Add all the elements of the list 'o' at the end of the
  * list 'l'. The list 'other' remains empty but otherwise valid. */
 void listJoin(list *l, list *o) {

src/adlist.h

@@ -88,7 +88,8 @@ listNode *listSearchKey(list *list, void *key);
 listNode *listIndex(list *list, PORT_LONG index);
 void listRewind(list *list, listIter *li);
 void listRewindTail(list *list, listIter *li);
-void listRotate(list *list);
+void listRotateTailToHead(list *list);
+void listRotateHeadToTail(list *list);
 void listJoin(list *l, list *o);
 
 /* Directions for iterators */

src/blocked.c

@@ -67,6 +67,21 @@
 
 int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int where);
 
+/* This structure represents the blocked key information that we store
+ * in the client structure. Each client blocked on keys, has a
+ * client->bpop.keys hash table. The keys of the hash table are Redis
+ * keys pointers to 'robj' structures. The value is this structure.
+ * The structure has two goals: firstly we store the list node that this
+ * client uses to be listed in the database "blocked clients for this key"
+ * list, so we can later unblock in O(1) without a list scan.
+ * Secondly for certain blocking types, we have additional info. Right now
+ * the only use for additional info we have is when clients are blocked
+ * on streams, as we have to remember the ID it blocked for. */
+typedef struct bkinfo {
+    listNode *listnode;     /* List node for db->blocking_keys[key] list. */
+    streamID stream_id;     /* Stream ID if we blocked in a stream. */
+} bkinfo;
+
 /* Get a timeout value from an object and store it into 'timeout'.
  * The final timeout is always stored as milliseconds as a time where the
  * timeout will expire, however the parsing is performed according to
@@ -291,8 +306,7 @@ void handleClientsBlockedOnKeys(void) {
                         if (receiver->btype != BLOCKED_LIST) {
                             /* Put at the tail, so that at the next call
                              * we'll not run into it again. */
-                            listDelNode(clients,clientnode);
-                            listAddNodeTail(clients,receiver);
+                            listRotateHeadToTail(clients);
                             continue;
                         }
 
@@ -353,8 +367,7 @@ void handleClientsBlockedOnKeys(void) {
                         if (receiver->btype != BLOCKED_ZSET) {
                             /* Put at the tail, so that at the next call
                              * we'll not run into it again. */
-                            listDelNode(clients,clientnode);
-                            listAddNodeTail(clients,receiver);
+                            listRotateHeadToTail(clients);
                             continue;
                         }
 
@@ -398,8 +411,9 @@ void handleClientsBlockedOnKeys(void) {
                     while((ln = listNext(&li))) {
                         client *receiver = listNodeValue(ln);
                         if (receiver->btype != BLOCKED_STREAM) continue;
-                        streamID *gt = dictFetchValue(receiver->bpop.keys,
-                                                      rl->key);
+                        bkinfo *bki = dictFetchValue(receiver->bpop.keys,
+                                                     rl->key);
+                        streamID *gt = &bki->stream_id;
 
                         /* If we blocked in the context of a consumer
                          * group, we need to resolve the group and update the
@@ -516,17 +530,15 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo
     if (target != NULL) incrRefCount(target);
 
     for (j = 0; j < numkeys; j++) {
-        /* The value associated with the key name in the bpop.keys dictionary
-         * is NULL for lists and sorted sets, or the stream ID for streams. */
-        void *key_data = NULL;
-        if (btype == BLOCKED_STREAM) {
-            key_data = zmalloc(sizeof(streamID));
-            memcpy(key_data,ids+j,sizeof(streamID));
-        }
+        /* Allocate our bkinfo structure, associated to each key the client
+         * is blocked for. */
+        bkinfo *bki = zmalloc(sizeof(*bki));
+        if (btype == BLOCKED_STREAM)
+            bki->stream_id = ids[j];
 
         /* If the key already exists in the dictionary ignore it. */
-        if (dictAdd(c->bpop.keys,keys[j],key_data) != DICT_OK) {
-            zfree(key_data);
+        if (dictAdd(c->bpop.keys,keys[j],bki) != DICT_OK) {
+            zfree(bki);
             continue;
         }
         incrRefCount(keys[j]);
@@ -545,6 +557,7 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo
             l = dictGetVal(de);
         }
         listAddNodeTail(l,c);
+        bki->listnode = listLast(l);
     }
     blockClient(c,btype);
 }
@@ -561,11 +574,12 @@ void unblockClientWaitingData(client *c) {
     /* The client may wait for multiple keys, so unblock it for every key. */
     while((de = dictNext(di)) != NULL) {
         robj *key = dictGetKey(de);
+        bkinfo *bki = dictGetVal(de);
 
         /* Remove this client from the list of clients waiting for this key. */
         l = dictFetchValue(c->db->blocking_keys,key);
         serverAssertWithInfo(c,key,l != NULL);
-        listDelNode(l,listSearchKey(l,c));
+        listDelNode(l,bki->listnode);
         /* If the list is empty we need to remove it to avoid wasting memory */
         if (listLength(l) == 0)
             dictDelete(c->db->blocking_keys,key);

src/server.c

@@ -998,7 +998,7 @@ void clientsCron(void) {
         /* Rotate the list, take the current head, process.
          * This way if the client must be removed from the list it's the
          * first element and we don't incur into O(N) computation. */
-        listRotate(server.clients);
+        listRotateTailToHead(server.clients);
         head = listFirst(server.clients);
         c = listNodeValue(head);
         /* The following functions do different service checks on the client.
@@ -2416,8 +2416,13 @@ struct redisCommand *lookupCommandOrOriginal(sds name) {
  * + PROPAGATE_AOF (propagate into the AOF file if is enabled)
  * + PROPAGATE_REPL (propagate into the replication link)
  *
- * This should not be used inside commands implementation. Use instead
- * alsoPropagate(), preventCommandPropagation(), forceCommandPropagation().
+ * This should not be used inside commands implementation since it will not
+ * wrap the resulting commands in MULTI/EXEC. Use instead alsoPropagate(),
+ * preventCommandPropagation(), forceCommandPropagation().
+ *
+ * However for functions that need to (also) propagate out of the context of a
+ * command execution, for example when serving a blocked client, you
+ * want to use propagate().
  */
 void propagate(struct redisCommand *cmd, int dbid, robj **argv, int argc,
                int flags)

src/t_stream.c

@@ -842,7 +842,7 @@ void streamPropagateXCLAIM(client *c, robj *key, streamCG *group, robj *groupnam
     argv[11] = createStringObject("JUSTID",6);
     argv[12] = createStringObject("LASTID",6);
     argv[13] = createObjectFromStreamID(&group->last_id);
-    alsoPropagate(server.xclaimCommand,c->db->id,argv,14,PROPAGATE_AOF|PROPAGATE_REPL);
+    propagate(server.xclaimCommand,c->db->id,argv,14,PROPAGATE_AOF|PROPAGATE_REPL);
     decrRefCount(argv[0]);
     decrRefCount(argv[3]);
     decrRefCount(argv[4]);

src/version.h

@@ -1 +1 @@
-#define REDIS_VERSION "5.0.8"
+#define REDIS_VERSION "5.0.9"