Judgedaemon receives 500 with SQL `Serialization failure: 1213 Deadlock found` when rejudging is created

[eldering]

Description of the problem

While a rejudging was being created, I got this error on a judgedaemon

[Oct 26 08:56:55.061] judgedaemon[16648]: warning: Error while executing curl POST to url http://localhost/domjudge/api/judgehosts/fetch-work: http status code: 500, request size = 36, response: array (
  'code' => 500,
  'message' => 'An exception occurred while executing a query: SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction',
  'class' => 'Doctrine\\DBAL\\Exception\\DeadlockException',
  'trace' => 
  array (
    0 => 
    array (
      'namespace' => '',
      'short_class' => '',
      'class' => '',
      'type' => '',
      'function' => '',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/vendor/doctrine/dbal/src/Driver/API/MySQL/ExceptionConverter.php',
      'line' => 39,
      'args' => 
      array (
      ),
    ),
    1 => 
    array (
      'namespace' => 'Doctrine\\DBAL\\Driver\\API\\MySQL',
      'short_class' => 'ExceptionConverter',
      'class' => 'Doctrine\\DBAL\\Driver\\API\\MySQL\\ExceptionConverter',
      'type' => '->',
      'function' => 'convert',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/vendor/doctrine/dbal/src/Connection.php',
      'line' => 1939,
      'args' => 
      array (
      ),
    ),
    2 => 
    array (
      'namespace' => 'Doctrine\\DBAL',
      'short_class' => 'Connection',
      'class' => 'Doctrine\\DBAL\\Connection',
      'type' => '->',
      'function' => 'handleDriverException',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/vendor/doctrine/dbal/src/Connection.php',
      'line' => 1881,
      'args' => 
      array (
      ),
    ),
    3 => 
    array (
      'namespace' => 'Doctrine\\DBAL',
      'short_class' => 'Connection',
      'class' => 'Doctrine\\DBAL\\Connection',
      'type' => '->',
      'function' => 'convertExceptionDuringQuery',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/vendor/doctrine/dbal/src/Connection.php',
      'line' => 1213,
      'args' => 
      array (
      ),
    ),
    4 => 
    array (
      'namespace' => 'Doctrine\\DBAL',
      'short_class' => 'Connection',
      'class' => 'Doctrine\\DBAL\\Connection',
      'type' => '->',
      'function' => 'executeStatement',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/src/Controller/API/JudgehostController.php',
      'line' => 1707,
      'args' => 
      array (
      ),
    ),
    5 => 
    array (
      'namespace' => 'App\\Controller\\API',
      'short_class' => 'JudgehostController',
      'class' => 'App\\Controller\\API\\JudgehostController',
      'type' => '->',
      'function' => 'serializeJudgeTasks',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/src/Controller/API/JudgehostController.php',
      'line' => 1781,
      'args' => 
      array (
      ),
    ),
    6 => 
    array (
      'namespace' => 'App\\Controller\\API',
      'short_class' => 'JudgehostController',
      'class' => 'App\\Controller\\API\\JudgehostController',
      'type' => '->',
      'function' => 'getJudgetasks',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/src/Controller/API/JudgehostController.php',
      'line' => 1594,
      'args' => 
      array (
      ),
    ),
    7 => 
    array (
      'namespace' => 'App\\Controller\\API',
      'short_class' => 'JudgehostController',
      'class' => 'App\\Controller\\API\\JudgehostController',
      'type' => '->',
      'function' => 'App\\Controller\\API\\{closure}',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/vendor/doctrine/orm/src/EntityManager.php',
      'line' => 272,
      'args' => 
      array (
      ),
    ),
    8 => 
    array (
      'namespace' => 'Doctrine\\ORM',
      'short_class' => 'EntityManager',
      'class' => 'Doctrine\\ORM\\EntityManager',
      'type' => '->',
      'function' => 'wrapInTransaction',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/src/Controller/API/JudgehostController.php',
      'line' => 1583,
      'args' => 
      array (
      ),
    ),
    9 => 
    array (
      'namespace' => 'App\\Controller\\API',
      'short_class' => 'JudgehostController',
      'class' => 'App\\Controller\\API\\JudgehostController',
      'type' => '->',
      'function' => 'getJudgeTasksAction',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/vendor/symfony/http-kernel/HttpKernel.php',
      'line' => 181,
      'args' => 
      array (
      ),
    ),
    10 => 
    array (
      'namespace' => 'Symfony\\Component\\HttpKernel',
      'short_class' => 'HttpKernel',
      'class' => 'Symfony\\Component\\HttpKernel\\HttpKernel',
      'type' => '->',
      'function' => 'handleRaw',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/vendor/symfony/http-kernel/HttpKernel.php',
      'line' => 76,
      'args' => 
      array (
      ),
    ),
    11 => 
    array (
      'namespace' => 'Symfony\\Component\\HttpKernel',
      'short_class' => 'HttpKernel',
      'class' => 'Symfony\\Component\\HttpKernel\\HttpKernel',
      'type' => '->',
      'function' => 'handle',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/vendor/symfony/http-kernel/Kernel.php',
      'line' => 197,
      'args' => 
      array (
      ),
    ),
    12 => 
    array (
      'namespace' => 'Symfony\\Component\\HttpKernel',
      'short_class' => 'Kernel',
      'class' => 'Symfony\\Component\\HttpKernel\\Kernel',
      'type' => '->',
      'function' => 'handle',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/vendor/symfony/runtime/Runner/Symfony/HttpKernelRunner.php',
      'line' => 35,
      'args' => 
      array (
      ),
    ),
    13 => 
    array (
      'namespace' => 'Symfony\\Component\\Runtime\\Runner\\Symfony',
      'short_class' => 'HttpKernelRunner',
      'class' => 'Symfony\\Component\\Runtime\\Runner\\Symfony\\HttpKernelRunner',
      'type' => '->',
      'function' => 'run',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/vendor/autoload_runtime.php',
      'line' => 29,
      'args' => 
      array (
      ),
    ),
    14 => 
    array (
      'namespace' => '',
      'short_class' => '',
      'class' => '',
      'type' => '',
      'function' => 'require_once',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/public/index.php',
      'line' => 5,
      'args' => 
      array (
        0 => 
        array (
          0 => 'string',
          1 => '/home/jaap/domjudge/git/domjudge/webapp/vendor/autoload_runtime.php',
        ),
      ),
    ),
  ),
)

Your environment

• current main branch at 2d6f4ff
• Debian bookworm with cgroup v2
• DB dump from WF Astana

[eldering]

transaction isolation level is set to SERIALIZABLE.

[eldering]

I've also tried with isolation level set to the default REPEATABLE-READ and then again switched back to SERIALIZABLE. Now in both these cases I got a different error:

[Nov 22 15:19:01.433] judgedaemon[355984]: warning: Error while executing curl POST to url http://localhost/domjudge/api/judgehosts/add-judging-run/tiger-2/1259342: http status code: 500, request size = 22597, response: array (
  'code' => 500,
  'message' => 'internal bug: the evaluated result changed during judging',
  'class' => 'BadMethodCallException',
  'trace' => 
  array (
    0 => 
    array (
      'namespace' => '',
      'short_class' => '',
      'class' => '',
      'type' => '',
      'function' => '',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/src/Controller/API/JudgehostController.php',
      'line' => 1036,
      'args' => 
      array (
      ),
    ),
    1 => 
    array (
      'namespace' => 'App\\Controller\\API',
      'short_class' => 'JudgehostController',
      'class' => 'App\\Controller\\API\\JudgehostController',
      'type' => '->',
      'function' => 'addSingleJudgingRun',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/src/Controller/API/JudgehostController.php',
      'line' => 656,
      'args' => 
      array (
      ),
    ),
    2 => 
    array (
      'namespace' => 'App\\Controller\\API',
      'short_class' => 'JudgehostController',
      'class' => 'App\\Controller\\API\\JudgehostController',
      'type' => '->',
      'function' => 'addJudgingRunAction',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/vendor/symfony/http-kernel/HttpKernel.php',
      'line' => 181,
      'args' => 
      array (
      ),
    ),
    3 => 
    array (
      'namespace' => 'Symfony\\Component\\HttpKernel',
      'short_class' => 'HttpKernel',
      'class' => 'Symfony\\Component\\HttpKernel\\HttpKernel',
      'type' => '->',
      'function' => 'handleRaw',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/vendor/symfony/http-kernel/HttpKernel.php',
      'line' => 76,
      'args' => 
      array (
      ),
    ),
    4 => 
    array (
      'namespace' => 'Symfony\\Component\\HttpKernel',
      'short_class' => 'HttpKernel',
      'class' => 'Symfony\\Component\\HttpKernel\\HttpKernel',
      'type' => '->',
      'function' => 'handle',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/vendor/symfony/http-kernel/Kernel.php',
      'line' => 197,
      'args' => 
      array (
      ),
    ),
    5 => 
    array (
      'namespace' => 'Symfony\\Component\\HttpKernel',
      'short_class' => 'Kernel',
      'class' => 'Symfony\\Component\\HttpKernel\\Kernel',
      'type' => '->',
      'function' => 'handle',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/vendor/symfony/runtime/Runner/Symfony/HttpKernelRunner.php',
      'line' => 35,
      'args' => 
      array (
      ),
    ),
    6 => 
    array (
      'namespace' => 'Symfony\\Component\\Runtime\\Runner\\Symfony',
      'short_class' => 'HttpKernelRunner',
      'class' => 'Symfony\\Component\\Runtime\\Runner\\Symfony\\HttpKernelRunner',
      'type' => '->',
      'function' => 'run',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/vendor/autoload_runtime.php',
      'line' => 29,
      'args' => 
      array (
      ),
    ),
    7 => 
    array (
      'namespace' => '',
      'short_class' => '',
      'class' => '',
      'type' => '',
      'function' => 'require_once',
      'file' => '/home/jaap/domjudge/git/domjudge/webapp/public/index.php',
      'line' => 5,
      'args' => 
      array (
        0 => 
        array (
          0 => 'string',
          1 => '/home/jaap/domjudge/git/domjudge/webapp/vendor/autoload_runtime.php',
        ),
      ),
    ),
  ),
)