Skip to content

Commit fd8f3f7

Browse files
mythzmythz
committed
Add PoolTimeout and Console App to debug multi-threading issues with server terminated client connections
1 parent df3a9fd commit fd8f3f7

File tree

8 files changed

+307
-18
lines changed

8 files changed

+307
-18
lines changed

‎src/ServiceStack.Redis.sln

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11

2-
Microsoft Visual Studio Solution File, Format Version 11.00
3-
# Visual Studio 2010
2+
Microsoft Visual Studio Solution File, Format Version 12.00
3+
# Visual Studio 2012
44
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Build", "Build", "{38F69F8F-9303-4BAF-B081-D28339163E07}"
55
ProjectSection(SolutionItems) = preProject
66
..\build\build.bat = ..\build\build.bat
@@ -12,6 +12,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceStack.Redis", "Servi
1212
EndProject
1313
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceStack.Redis.Tests", "..\tests\ServiceStack.Redis.Tests\ServiceStack.Redis.Tests.csproj", "{951D28EE-5D22-4C62-AC0F-1661A8CEEC5A}"
1414
EndProject
15+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestMqHost", "TestMqHost\TestMqHost.csproj", "{B932C136-4365-4C37-8187-96703715EBB4}"
16+
EndProject
1517
Global
1618
GlobalSection(SolutionConfigurationPlatforms) = preSolution
1719
Debug|Any CPU = Debug|Any CPU
@@ -56,6 +58,18 @@ Global
5658
{951D28EE-5D22-4C62-AC0F-1661A8CEEC5A}.STATIC_ONLY NO_EXPRESSIONS|Any CPU.Build.0 = STATIC_ONLY NO_EXPRESSIONS|Any CPU
5759
{951D28EE-5D22-4C62-AC0F-1661A8CEEC5A}.STATIC_ONLY NO_EXPRESSIONS|x86.ActiveCfg = STATIC_ONLY NO_EXPRESSIONS|x86
5860
{951D28EE-5D22-4C62-AC0F-1661A8CEEC5A}.STATIC_ONLY NO_EXPRESSIONS|x86.Build.0 = STATIC_ONLY NO_EXPRESSIONS|x86
61+
{B932C136-4365-4C37-8187-96703715EBB4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
62+
{B932C136-4365-4C37-8187-96703715EBB4}.Debug|Any CPU.Build.0 = Debug|Any CPU
63+
{B932C136-4365-4C37-8187-96703715EBB4}.Debug|x86.ActiveCfg = Debug|Any CPU
64+
{B932C136-4365-4C37-8187-96703715EBB4}.MonoTouch|Any CPU.ActiveCfg = Release|Any CPU
65+
{B932C136-4365-4C37-8187-96703715EBB4}.MonoTouch|Any CPU.Build.0 = Release|Any CPU
66+
{B932C136-4365-4C37-8187-96703715EBB4}.MonoTouch|x86.ActiveCfg = Release|Any CPU
67+
{B932C136-4365-4C37-8187-96703715EBB4}.Release|Any CPU.ActiveCfg = Release|Any CPU
68+
{B932C136-4365-4C37-8187-96703715EBB4}.Release|Any CPU.Build.0 = Release|Any CPU
69+
{B932C136-4365-4C37-8187-96703715EBB4}.Release|x86.ActiveCfg = Release|Any CPU
70+
{B932C136-4365-4C37-8187-96703715EBB4}.STATIC_ONLY NO_EXPRESSIONS|Any CPU.ActiveCfg = Release|Any CPU
71+
{B932C136-4365-4C37-8187-96703715EBB4}.STATIC_ONLY NO_EXPRESSIONS|Any CPU.Build.0 = Release|Any CPU
72+
{B932C136-4365-4C37-8187-96703715EBB4}.STATIC_ONLY NO_EXPRESSIONS|x86.ActiveCfg = Release|Any CPU
5973
EndGlobalSection
6074
GlobalSection(SolutionProperties) = preSolution
6175
HideSolutionNode = FALSE

‎src/ServiceStack.Redis/PooledRedisClientManager.cs

Lines changed: 47 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,11 @@ public partial class PooledRedisClientManager
3434
"Redis Timeout expired. The timeout period elapsed prior to obtaining a connection from the pool. This may have occurred because all pooled connections were in use.";
3535

3636
protected readonly int PoolSizeMultiplier = 10;
37-
public int? PoolTimeOut { get; set; }
37+
public int RecheckPoolAfterMs = 100;
38+
public int? PoolTimeout { get; set; }
3839
public int? ConnectTimeout { get; set; }
3940
public int? SocketSendTimeout { get; set; }
4041
public int? SocketReceiveTimeout { get; set; }
41-
public bool CheckConnected { get; set; }
4242

4343
/// <summary>
4444
/// Gets or sets object key prefix.
@@ -133,7 +133,7 @@ public PooledRedisClientManager(
133133
};
134134

135135
// if timeout provided, convert into milliseconds
136-
this.PoolTimeOut = poolTimeOutSeconds != null
136+
this.PoolTimeout = poolTimeOutSeconds != null
137137
? poolTimeOutSeconds * 1000
138138
: null;
139139

@@ -162,14 +162,14 @@ public IRedisClient GetClient()
162162
RedisClient inActiveClient;
163163
while ((inActiveClient = GetInActiveWriteClient()) == null)
164164
{
165-
if (PoolTimeOut.HasValue)
165+
if (PoolTimeout.HasValue)
166166
{
167167
// wait for a connection, cry out if made to wait too long
168-
if (!Monitor.Wait(writeClients, PoolTimeOut.Value))
168+
if (!Monitor.Wait(writeClients, PoolTimeout.Value))
169169
throw new TimeoutException(PoolTimeoutError);
170170
}
171171
else
172-
Monitor.Wait(writeClients);
172+
Monitor.Wait(writeClients, RecheckPoolAfterMs);
173173
}
174174

175175
WritePoolIndex++;
@@ -216,8 +216,7 @@ private RedisClient GetInActiveWriteClient()
216216
var nextHost = ReadWriteHosts[nextHostIndex];
217217
for (var i = nextHostIndex; i < writeClients.Length; i += ReadWriteHosts.Count)
218218
{
219-
if (writeClients[i] != null && !writeClients[i].Active && !writeClients[i].HadExceptions
220-
&& (!CheckConnected || writeClients[i].IsSocketConnected()))
219+
if (writeClients[i] != null && !writeClients[i].Active && !writeClients[i].HadExceptions)
221220
return writeClients[i];
222221
else if (writeClients[i] == null || writeClients[i].HadExceptions)
223222
{
@@ -255,8 +254,15 @@ public virtual IRedisClient GetReadOnlyClient()
255254
RedisClient inActiveClient;
256255
while ((inActiveClient = GetInActiveReadClient()) == null)
257256
{
258-
Monitor.Wait(readClients);
259-
}
257+
if (PoolTimeout.HasValue)
258+
{
259+
// wait for a connection, cry out if made to wait too long
260+
if (!Monitor.Wait(readClients, PoolTimeout.Value))
261+
throw new TimeoutException(PoolTimeoutError);
262+
}
263+
else
264+
Monitor.Wait(readClients, RecheckPoolAfterMs);
265+
}
260266

261267
ReadPoolIndex++;
262268
inActiveClient.Active = true;
@@ -302,8 +308,7 @@ private RedisClient GetInActiveReadClient()
302308
var nextHost = ReadOnlyHosts[nextHostIndex];
303309
for (var i = nextHostIndex; i < readClients.Length; i += ReadOnlyHosts.Count)
304310
{
305-
if (readClients[i] != null && !readClients[i].Active && !readClients[i].HadExceptions
306-
&& (!CheckConnected || readClients[i].IsSocketConnected()))
311+
if (readClients[i] != null && !readClients[i].Active && !readClients[i].HadExceptions)
307312
return readClients[i];
308313
else if (readClients[i] == null || readClients[i].HadExceptions)
309314
{
@@ -477,6 +482,36 @@ private void AssertValidReadOnlyPool()
477482
throw new InvalidOperationException("Need a minimum read pool size of 1, then call Start()");
478483
}
479484

485+
public int[] GetClientPoolActiveStates()
486+
{
487+
var activeStates = new int[writeClients.Length];
488+
lock (writeClients)
489+
{
490+
for (int i = 0; i < writeClients.Length; i++)
491+
{
492+
activeStates[i] = writeClients[i] == null
493+
? -1
494+
: writeClients[i].Active ? 1 : 0;
495+
}
496+
}
497+
return activeStates;
498+
}
499+
500+
public int[] GetReadOnlyClientPoolActiveStates()
501+
{
502+
var activeStates = new int[readClients.Length];
503+
lock (readClients)
504+
{
505+
for (int i = 0; i < readClients.Length; i++)
506+
{
507+
activeStates[i] = readClients[i] == null
508+
? -1
509+
: readClients[i].Active ? 1 : 0;
510+
}
511+
}
512+
return activeStates;
513+
}
514+
480515
~PooledRedisClientManager()
481516
{
482517
Dispose(false);

‎src/ServiceStack.Redis/ShardedConnectionPool.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ public class ShardedConnectionPool : PooledRedisClientManager
2626
public ShardedConnectionPool(string name, int weight, params string[] readWriteHosts)
2727
: base(readWriteHosts)
2828
{
29-
this.PoolTimeOut = 1000;
29+
this.PoolTimeout = 1000;
3030
this.name = name;
3131
this.weight = weight;
3232
}

‎src/TestMqHost/App.config

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
<?xml version="1.0" encoding="utf-8" ?>
2+
<configuration>
3+
<startup>
4+
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
5+
</startup>
6+
</configuration>

‎src/TestMqHost/Program.cs

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
using System;
2+
using System.Threading;
3+
using ServiceStack.Common;
4+
using ServiceStack.Logging;
5+
using ServiceStack.Logging.Support.Logging;
6+
using ServiceStack.Redis;
7+
using ServiceStack.Redis.Messaging;
8+
using ServiceStack.Text;
9+
10+
namespace TestMqHost
11+
{
12+
public class Incr
13+
{
14+
public int Value { get; set; }
15+
}
16+
17+
class Program
18+
{
19+
static void Main(string[] args)
20+
{
21+
22+
//LogManager.LogFactory = new ConsoleLogFactory();
23+
24+
var clientManager = new PooledRedisClientManager(new[] { "localhost" })
25+
{
26+
PoolTimeout = 1000,
27+
};
28+
29+
var mqHost = new RedisMqServer(clientManager, retryCount: 2);
30+
31+
var sum = 0;
32+
mqHost.RegisterHandler<Incr>(c =>
33+
{
34+
var dto = c.GetBody();
35+
sum += dto.Value;
36+
"Received {0}, sum: {1}".Print(dto.Value, sum);
37+
return null;
38+
});
39+
40+
mqHost.Start();
41+
42+
10.Times(i =>
43+
{
44+
ThreadPool.QueueUserWorkItem(x =>
45+
{
46+
using (var client = mqHost.CreateMessageQueueClient())
47+
{
48+
try
49+
{
50+
lock (clientManager)
51+
"Publish: {0}...".Print(i);
52+
client.Publish(new Incr { Value = i });
53+
}
54+
catch (Exception ex)
55+
{
56+
lock (clientManager)
57+
"Start Publish exception: {0}".Print(ex.Message);
58+
clientManager.GetClientPoolActiveStates().PrintDump();
59+
clientManager.GetReadOnlyClientPoolActiveStates().PrintDump();
60+
}
61+
Thread.Sleep(10);
62+
}
63+
});
64+
});
65+
66+
ThreadPool.QueueUserWorkItem(_ =>
67+
{
68+
using (var client = (RedisClient)clientManager.GetClient())
69+
{
70+
client.SetConfig("timeout", "1");
71+
var clientAddrs = client.GetClientList().ConvertAll(x => x["addr"]);
72+
lock (clientManager)
73+
"Killing clients: {0}...".Print(clientAddrs.Dump());
74+
try
75+
{
76+
clientAddrs.ForEach(client.ClientKill);
77+
}
78+
catch (Exception ex)
79+
{
80+
lock (clientManager)
81+
"Client exception: {0}".Print(ex.Message);
82+
}
83+
}
84+
});
85+
86+
20.Times(i =>
87+
{
88+
using (var client = mqHost.CreateMessageQueueClient())
89+
{
90+
try
91+
{
92+
lock (clientManager)
93+
"Publish: {0}...".Print(i);
94+
client.Publish(new Incr { Value = i });
95+
}
96+
catch (Exception ex)
97+
{
98+
lock (clientManager)
99+
"Publish exception: {0}".Print(ex.Message);
100+
clientManager.GetClientPoolActiveStates().PrintDump();
101+
clientManager.GetReadOnlyClientPoolActiveStates().PrintDump();
102+
}
103+
}
104+
105+
Thread.Sleep(1000);
106+
});
107+
}
108+
}
109+
}

‎src/TestMqHost/Properties/AssemblyInfo.cs

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
using System.Reflection;
2+
using System.Runtime.CompilerServices;
3+
using System.Runtime.InteropServices;
4+
5+
// General Information about an assembly is controlled through the following
6+
// set of attributes. Change these attribute values to modify the information
7+
// associated with an assembly.
8+
[assembly: AssemblyTitle("TestMqHost")]
9+
[assembly: AssemblyDescription("")]
10+
[assembly: AssemblyConfiguration("")]
11+
[assembly: AssemblyCompany("Microsoft")]
12+
[assembly: AssemblyProduct("TestMqHost")]
13+
[assembly: AssemblyCopyright("Copyright © Microsoft 2013")]
14+
[assembly: AssemblyTrademark("")]
15+
[assembly: AssemblyCulture("")]
16+
17+
// Setting ComVisible to false makes the types in this assembly not visible
18+
// to COM components. If you need to access a type in this assembly from
19+
// COM, set the ComVisible attribute to true on that type.
20+
[assembly: ComVisible(false)]
21+
22+
// The following GUID is for the ID of the typelib if this project is exposed to COM
23+
[assembly: Guid("6df3bc57-6380-4f2f-8d02-731f04b2aec7")]
24+
25+
// Version information for an assembly consists of the following four values:
26+
//
27+
// Major Version
28+
// Minor Version
29+
// Build Number
30+
// Revision
31+
//
32+
// You can specify all the values or you can default the Build and Revision Numbers
33+
// by using the '*' as shown below:
34+
// [assembly: AssemblyVersion("1.0.*")]
35+
[assembly: AssemblyVersion("1.0.0.0")]
36+
[assembly: AssemblyFileVersion("1.0.0.0")]

‎src/TestMqHost/TestMqHost.csproj

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
4+
<PropertyGroup>
5+
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
6+
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
7+
<ProjectGuid>{B932C136-4365-4C37-8187-96703715EBB4}</ProjectGuid>
8+
<OutputType>Exe</OutputType>
9+
<AppDesignerFolder>Properties</AppDesignerFolder>
10+
<RootNamespace>TestMqHost</RootNamespace>
11+
<AssemblyName>TestMqHost</AssemblyName>
12+
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
13+
<FileAlignment>512</FileAlignment>
14+
</PropertyGroup>
15+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
16+
<PlatformTarget>AnyCPU</PlatformTarget>
17+
<DebugSymbols>true</DebugSymbols>
18+
<DebugType>full</DebugType>
19+
<Optimize>false</Optimize>
20+
<OutputPath>bin\Debug\</OutputPath>
21+
<DefineConstants>DEBUG;TRACE</DefineConstants>
22+
<ErrorReport>prompt</ErrorReport>
23+
<WarningLevel>4</WarningLevel>
24+
</PropertyGroup>
25+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
26+
<PlatformTarget>AnyCPU</PlatformTarget>
27+
<DebugType>pdbonly</DebugType>
28+
<Optimize>true</Optimize>
29+
<OutputPath>bin\Release\</OutputPath>
30+
<DefineConstants>TRACE</DefineConstants>
31+
<ErrorReport>prompt</ErrorReport>
32+
<WarningLevel>4</WarningLevel>
33+
</PropertyGroup>
34+
<ItemGroup>
35+
<Reference Include="ServiceStack.Common, Version=3.9.49.0, Culture=neutral, processorArchitecture=MSIL">
36+
<SpecificVersion>False</SpecificVersion>
37+
<HintPath>..\..\lib\ServiceStack.Common.dll</HintPath>
38+
</Reference>
39+
<Reference Include="ServiceStack.Interfaces, Version=3.9.0.0, Culture=neutral, processorArchitecture=MSIL">
40+
<SpecificVersion>False</SpecificVersion>
41+
<HintPath>..\..\lib\ServiceStack.Interfaces.dll</HintPath>
42+
</Reference>
43+
<Reference Include="ServiceStack.Text, Version=3.9.45.0, Culture=neutral, processorArchitecture=MSIL">
44+
<SpecificVersion>False</SpecificVersion>
45+
<HintPath>..\..\lib\ServiceStack.Text.dll</HintPath>
46+
</Reference>
47+
<Reference Include="System" />
48+
<Reference Include="System.Core" />
49+
<Reference Include="System.Xml.Linq" />
50+
<Reference Include="System.Data.DataSetExtensions" />
51+
<Reference Include="Microsoft.CSharp" />
52+
<Reference Include="System.Data" />
53+
<Reference Include="System.Xml" />
54+
</ItemGroup>
55+
<ItemGroup>
56+
<Compile Include="Program.cs" />
57+
<Compile Include="Properties\AssemblyInfo.cs" />
58+
</ItemGroup>
59+
<ItemGroup>
60+
<None Include="App.config" />
61+
</ItemGroup>
62+
<ItemGroup>
63+
<ProjectReference Include="..\ServiceStack.Redis\ServiceStack.Redis.csproj">
64+
<Project>{af99f19b-4c04-4f58-81ef-b092f1fcc540}</Project>
65+
<Name>ServiceStack.Redis</Name>
66+
</ProjectReference>
67+
</ItemGroup>
68+
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
69+
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
70+
Other similar extension points exist, see Microsoft.Common.targets.
71+
<Target Name="BeforeBuild">
72+
</Target>
73+
<Target Name="AfterBuild">
74+
</Target>
75+
-->
76+
</Project>

0 commit comments

Comments
 (0)