Skip to content

Commit 129a0d7

Browse files
taylorotwellStyleCIBot
taylorotwell
and
StyleCIBot
authored
More scalable Redis cache tags (#45690)
* initial work on more scalable redis cache tags * Apply fixes from StyleCI * add integration test * update tests * remove commented code * use nx flag Co-authored-by: StyleCI Bot <bot@styleci.io>
1 parent 89ffde6 commit 129a0d7

File tree

7 files changed

+259
-206
lines changed

7 files changed

+259
-206
lines changed

‎.github/workflows/tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ jobs:
2727
- 33306:3306
2828
options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
2929
redis:
30-
image: redis:5.0
30+
image: redis:7.0
3131
ports:
3232
- 6379:6379
3333
options: --entrypoint redis-server

‎docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ services:
2121
- "3306:3306"
2222
restart: always
2323
redis:
24-
image: redis:5.0-alpine
24+
image: redis:7.0-alpine
2525
ports:
2626
- "6379:6379"
2727
restart: always

‎src/Illuminate/Cache/RedisStore.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -244,7 +244,7 @@ public function flush()
244244
public function tags($names)
245245
{
246246
return new RedisTaggedCache(
247-
$this, new TagSet($this, is_array($names) ? $names : func_get_args())
247+
$this, new RedisTagSet($this, is_array($names) ? $names : func_get_args())
248248
);
249249
}
250250

‎src/Illuminate/Cache/RedisTagSet.php‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
<?php
2+
3+
namespace Illuminate\Cache;
4+
5+
use Illuminate\Support\LazyCollection;
6+
7+
class RedisTagSet extends TagSet
8+
{
9+
/**
10+
* Add a reference entry to the tag set's underlying sorted set.
11+
*
12+
* @param string $key
13+
* @param int $ttl
14+
* @param string $updateWhen
15+
* @return void
16+
*/
17+
public function addEntry(string $key, int $ttl = 0, $updateWhen = null)
18+
{
19+
$ttl = $ttl > 0 ? now()->addSeconds($ttl)->getTimestamp() : -1;
20+
21+
foreach ($this->tagIds() as $tagKey) {
22+
if ($updateWhen) {
23+
$this->store->connection()->zadd($this->store->getPrefix().$tagKey, $updateWhen, $ttl, $key);
24+
} else {
25+
$this->store->connection()->zadd($this->store->getPrefix().$tagKey, $ttl, $key);
26+
}
27+
}
28+
}
29+
30+
/**
31+
* Get all of the cache entry keys for the tag set.
32+
*
33+
* @return \Illuminate\Support\LazyCollection
34+
*/
35+
public function entries()
36+
{
37+
return LazyCollection::make(function () {
38+
foreach ($this->tagIds() as $tagKey) {
39+
$cursor = $defaultCursorValue = '0';
40+
41+
do {
42+
[$cursor, $entries] = $this->store->connection()->zscan(
43+
$this->store->getPrefix().$tagKey,
44+
$cursor,
45+
['match' => '*', 'count' => 1000]
46+
);
47+
48+
if (! is_array($entries)) {
49+
break;
50+
}
51+
52+
$entries = array_unique(array_keys($entries));
53+
54+
if (count($entries) === 0) {
55+
continue;
56+
}
57+
58+
foreach ($entries as $entry) {
59+
yield $entry;
60+
}
61+
} while (((string) $cursor) !== $defaultCursorValue);
62+
}
63+
});
64+
}
65+
66+
/**
67+
* Remove the stale entries from the tag set.
68+
*
69+
* @return void
70+
*/
71+
public function flushStaleEntries()
72+
{
73+
$this->store->connection()->pipeline(function ($pipe) {
74+
foreach ($this->tagIds() as $tagKey) {
75+
$pipe->zremrangebyscore($this->store->getPrefix().$tagKey, 0, now()->getTimestamp());
76+
}
77+
});
78+
}
79+
80+
/**
81+
* Flush the tag from the cache.
82+
*
83+
* @param string $name
84+
*/
85+
public function flushTag($name)
86+
{
87+
return $this->resetTag($name);
88+
}
89+
90+
/**
91+
* Reset the tag and return the new tag identifier.
92+
*
93+
* @param string $name
94+
* @return string
95+
*/
96+
public function resetTag($name)
97+
{
98+
$this->store->forget($this->tagKey($name));
99+
100+
return $this->tagId($name);
101+
}
102+
103+
/**
104+
* Get the unique tag identifier for a given tag.
105+
*
106+
* @param string $name
107+
* @return string
108+
*/
109+
public function tagId($name)
110+
{
111+
return "tag:{$name}:entries";
112+
}
113+
114+
/**
115+
* Get the tag identifier key for a given tag.
116+
*
117+
* @param string $name
118+
* @return string
119+
*/
120+
public function tagKey($name)
121+
{
122+
return "tag:{$name}:entries";
123+
}
124+
}

‎src/Illuminate/Cache/RedisTaggedCache.php‎

Lines changed: 34 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,22 @@
55
class RedisTaggedCache extends TaggedCache
66
{
77
/**
8-
* Forever reference key.
8+
* Store an item in the cache if the key does not exist.
99
*
10-
* @var string
10+
* @param string $key
11+
* @param mixed $value
12+
* @param \DateTimeInterface|\DateInterval|int|null $ttl
13+
* @return bool
1114
*/
12-
const REFERENCE_KEY_FOREVER = 'forever_ref';
15+
public function add($key, $value, $ttl = null)
16+
{
17+
$this->tags->addEntry(
18+
$this->itemKey($key),
19+
! is_null($ttl) ? $this->getSeconds($ttl) : 0
20+
);
1321

14-
/**
15-
* Standard reference key.
16-
*
17-
* @var string
18-
*/
19-
const REFERENCE_KEY_STANDARD = 'standard_ref';
22+
return parent::add($key, $value, $ttl);
23+
}
2024

2125
/**
2226
* Store an item in the cache.
@@ -28,11 +32,14 @@ class RedisTaggedCache extends TaggedCache
2832
*/
2933
public function put($key, $value, $ttl = null)
3034
{
31-
if ($ttl === null) {
35+
if (is_null($ttl)) {
3236
return $this->forever($key, $value);
3337
}
3438

35-
$this->pushStandardKeys($this->tags->getNamespace(), $key);
39+
$this->tags->addEntry(
40+
$this->itemKey($key),
41+
$this->getSeconds($ttl)
42+
);
3643

3744
return parent::put($key, $value, $ttl);
3845
}
@@ -46,7 +53,7 @@ public function put($key, $value, $ttl = null)
4653
*/
4754
public function increment($key, $value = 1)
4855
{
49-
$this->pushStandardKeys($this->tags->getNamespace(), $key);
56+
$this->tags->addEntry($this->itemKey($key), updateWhen: 'NX');
5057

5158
return parent::increment($key, $value);
5259
}
@@ -60,7 +67,7 @@ public function increment($key, $value = 1)
6067
*/
6168
public function decrement($key, $value = 1)
6269
{
63-
$this->pushStandardKeys($this->tags->getNamespace(), $key);
70+
$this->tags->addEntry($this->itemKey($key), updateWhen: 'NX');
6471

6572
return parent::decrement($key, $value);
6673
}
@@ -74,7 +81,7 @@ public function decrement($key, $value = 1)
7481
*/
7582
public function forever($key, $value)
7683
{
77-
$this->pushForeverKeys($this->tags->getNamespace(), $key);
84+
$this->tags->addEntry($this->itemKey($key));
7885

7986
return parent::forever($key, $value);
8087
}
@@ -86,129 +93,37 @@ public function forever($key, $value)
8693
*/
8794
public function flush()
8895
{
89-
$this->deleteForeverKeys();
90-
$this->deleteStandardKeys();
91-
96+
$this->flushValues();
9297
$this->tags->flush();
9398

9499
return true;
95100
}
96101

97102
/**
98-
* Store standard key references into store.
99-
*
100-
* @param string $namespace
101-
* @param string $key
102-
* @return void
103-
*/
104-
protected function pushStandardKeys($namespace, $key)
105-
{
106-
$this->pushKeys($namespace, $key, self::REFERENCE_KEY_STANDARD);
107-
}
108-
109-
/**
110-
* Store forever key references into store.
111-
*
112-
* @param string $namespace
113-
* @param string $key
114-
* @return void
115-
*/
116-
protected function pushForeverKeys($namespace, $key)
117-
{
118-
$this->pushKeys($namespace, $key, self::REFERENCE_KEY_FOREVER);
119-
}
120-
121-
/**
122-
* Store a reference to the cache key against the reference key.
123-
*
124-
* @param string $namespace
125-
* @param string $key
126-
* @param string $reference
127-
* @return void
128-
*/
129-
protected function pushKeys($namespace, $key, $reference)
130-
{
131-
$fullKey = $this->store->getPrefix().sha1($namespace).':'.$key;
132-
133-
foreach (explode('|', $namespace) as $segment) {
134-
$this->store->connection()->sadd($this->referenceKey($segment, $reference), $fullKey);
135-
}
136-
}
137-
138-
/**
139-
* Delete all of the items that were stored forever.
103+
* Flush the individual cache entries for the tags.
140104
*
141105
* @return void
142106
*/
143-
protected function deleteForeverKeys()
107+
protected function flushValues()
144108
{
145-
$this->deleteKeysByReference(self::REFERENCE_KEY_FOREVER);
146-
}
147-
148-
/**
149-
* Delete all standard items.
150-
*
151-
* @return void
152-
*/
153-
protected function deleteStandardKeys()
154-
{
155-
$this->deleteKeysByReference(self::REFERENCE_KEY_STANDARD);
156-
}
109+
$entries = $this->tags->entries()
110+
->map(fn (string $key) => $this->store->getPrefix().$key)
111+
->chunk(1000);
157112

158-
/**
159-
* Find and delete all of the items that were stored against a reference.
160-
*
161-
* @param string $reference
162-
* @return void
163-
*/
164-
protected function deleteKeysByReference($reference)
165-
{
166-
foreach (explode('|', $this->tags->getNamespace()) as $segment) {
167-
$this->deleteValues($segment = $this->referenceKey($segment, $reference));
168-
169-
$this->store->connection()->del($segment);
113+
foreach ($entries as $cacheKeys) {
114+
$this->store->connection()->del(...$cacheKeys);
170115
}
171116
}
172117

173118
/**
174-
* Delete item keys that have been stored against a reference.
119+
* Remove all stale reference entries from the tag set.
175120
*
176-
* @param string $referenceKey
177-
* @return void
121+
* @return bool
178122
*/
179-
protected function deleteValues($referenceKey)
123+
public function flushStale()
180124
{
181-
$cursor = $defaultCursorValue = '0';
182-
183-
do {
184-
[$cursor, $valuesChunk] = $this->store->connection()->sscan(
185-
$referenceKey, $cursor, ['match' => '*', 'count' => 1000]
186-
);
187-
188-
// PhpRedis client returns false if set does not exist or empty. Array destruction
189-
// on false stores null in each variable. If valuesChunk is null, it means that
190-
// there were not results from the previously executed "sscan" Redis command.
191-
if (is_null($valuesChunk)) {
192-
break;
193-
}
194-
195-
$valuesChunk = array_unique($valuesChunk);
196-
197-
if (count($valuesChunk) > 0) {
198-
$this->store->connection()->del(...$valuesChunk);
199-
}
200-
} while (((string) $cursor) !== $defaultCursorValue);
201-
}
125+
$this->tags->flushStaleEntries();
202126

203-
/**
204-
* Get the reference key for the segment.
205-
*
206-
* @param string $segment
207-
* @param string $suffix
208-
* @return string
209-
*/
210-
protected function referenceKey($segment, $suffix)
211-
{
212-
return $this->store->getPrefix().$segment.':'.$suffix;
127+
return true;
213128
}
214129
}

0 commit comments

Comments
 (0)