Indefinite wait when using encoding a secondary cmdbuf more than twice on D3D12.

[gdianaty]

Hello!

Attached you will find a modified version of Example_MultiThreading. This version has only one modification: pressing the Tab key will cause a re-encoding of both secondary command buffers.

Pressing tab two times will cause an indefinite hang on D3D12 in the function D3D12CommandContext::NextCommandAllocator(), within the usage of the LLGL convenience definition WaitForHigherSignal.

This bug does not happen on OpenGL or Direct3D11. Vulkan and Metal are unknown as I have not tested them.

/*
 * Example.cpp (Example_MultiThreading)
 *
 * Copyright (c) 2015 Lukas Hermanns. All rights reserved.
 * Licensed under the terms of the BSD 3-Clause license (see LICENSE.txt).
 */

#include <ExampleBase.h>
#include <LLGL/Utils/ForRange.h>
#include <thread>
#include <mutex>
#include <chrono>
#include <iomanip>


// Enables/disables the use of two secondary command buffers
#define ENABLE_SECONDARY_COMMAND_BUFFERS 1

class Measure
{

        using Clock     = std::chrono::system_clock;
        using TimePoint = std::chrono::time_point<Clock>;
        using Ticks     = std::chrono::milliseconds;

    public:

        // Interval (in milliseconds) to the next measurement result.
        Measure(std::uint64_t interval = 1000, const std::string& title = "Average Time") :
            interval_ { interval },
            title_    { title    }
        {
        }

        void Start()
        {
            // Start timer
            timer_.Start();
        }

        void Stop()
        {
            // Take sample
            elapsed_ += timer_.Stop();
            ++samples_;

            // Check if average elapsed time can be printed again
            auto end = Clock::now();
            auto diff = std::chrono::duration_cast<Ticks>(end - intervalStartTime_);

            if (diff.count() >= static_cast<long long>(interval_))
            {
                Print();
                intervalStartTime_ = Clock::now();
            }
        }

    private:

        void Print()
        {
            if (samples_ > 0)
            {
                auto averageTime = static_cast<double>(elapsed_);
                averageTime /= static_cast<double>(timer_.GetFrequency());
                averageTime *= 1000000.0;
                averageTime /= static_cast<double>(samples_);

                std::cout << title_ << ": ";
                std::cout << std::fixed << std::setprecision(6) << averageTime << " microseconds";
                std::cout << "         \r";
                std::flush(std::cout);

                samples_ = 0;
                elapsed_ = 0;
            }
        }

    private:

        Stopwatch       timer_;
        std::uint64_t   interval_           = 0;
        TimePoint       intervalStartTime_;
        std::uint64_t   samples_            = 0;
        std::uint64_t   elapsed_            = 0;
        std::string     title_;

};

constexpr std::uint32_t maxNumSwapBuffers = 3;

class Example_MultiThreading : public ExampleBase
{

    ShaderPipeline              shaderPipeline;
    LLGL::Buffer*               vertexBuffer                        = nullptr;
    LLGL::Buffer*               indexBuffer                         = nullptr;
    LLGL::PipelineLayout*       pipelineLayout                      = nullptr;
    LLGL::CommandBuffer*        primaryCmdBuffer[maxNumSwapBuffers] = {};

    std::uint32_t               numIndices                          = 0;
    std::mutex                  logMutex;

    Measure                     measure;

    struct Bundle
    {
        LLGL::PipelineState*    pipeline                            = nullptr;
        LLGL::Buffer*           constantBuffer                      = nullptr;
        LLGL::ResourceHeap*     resourceHeap                        = nullptr;
        LLGL::CommandBuffer*    secondaryCmdBuffer                  = nullptr;

        struct Matrices
        {
            Gs::Matrix4f        wvpMatrix;
            Gs::Matrix4f        wMatrix;
        }
        matrices;
    }
    bundle[2];

public:

    Example_MultiThreading() :
        ExampleBase { "LLGL Example: MultiThreading" }
    {
        auto vertexFormat = CreateBuffers();
        LoadShaders(vertexFormat);
        CreatePipelines();
        CreateCommandBuffers();
    }

private:

    LLGL::VertexFormat CreateBuffers()
    {
        // Specify vertex format
        LLGL::VertexFormat vertexFormat;
        vertexFormat.AppendAttribute({ "position", LLGL::Format::RGB32Float });
        vertexFormat.AppendAttribute({ "normal",   LLGL::Format::RGB32Float });
        vertexFormat.AppendAttribute({ "texCoord", LLGL::Format::RG32Float  });

        // Generate data for mesh buffers
        auto indices = GenerateTexturedCubeTriangleIndices();
        auto vertices = GenerateTexturedCubeVertices();
        numIndices = static_cast<std::uint32_t>(indices.size());

        // Create buffers for a simple 3D cube model
        vertexBuffer = CreateVertexBuffer(vertices, vertexFormat);
        indexBuffer = CreateIndexBuffer(indices, LLGL::Format::R32UInt);

        for (auto& bdl : bundle)
            bdl.constantBuffer = CreateConstantBuffer(bdl.matrices);

        return vertexFormat;
    }

    void LoadShaders(const LLGL::VertexFormat& vertexFormat)
    {
        // Load shader program
        shaderPipeline = LoadStandardShaderPipeline({ vertexFormat });
    }

    void CreatePipelines()
    {
        // Create pipeline layout
        pipelineLayout = renderer->CreatePipelineLayout(LLGL::Parse("heap{cbuffer(Scene@1):vert}"));

        // Create resource view heap
        for (auto& bdl : bundle)
            bdl.resourceHeap = renderer->CreateResourceHeap(pipelineLayout, { bdl.constantBuffer });

        // Setup graphics pipeline descriptors
        LLGL::GraphicsPipelineDescriptor pipelineDesc;
        {
            // Set references to shader program, and pipeline layout
            pipelineDesc.vertexShader                   = shaderPipeline.vs;
            pipelineDesc.fragmentShader                 = shaderPipeline.ps;
            pipelineDesc.pipelineLayout                 = pipelineLayout;
            pipelineDesc.rasterizer.multiSampleEnabled  = (GetSampleCount() > 1);

            // Enable depth test and writing
            pipelineDesc.depth.testEnabled              = true;
            pipelineDesc.depth.writeEnabled             = true;

            // Enable back-face culling
            pipelineDesc.rasterizer.cullMode            = LLGL::CullMode::Back;
        }

        // Create first graphics pipeline
        bundle[0].pipeline = renderer->CreatePipelineState(pipelineDesc);

        // Create second graphics pipeline
        {
            auto& targetDesc = pipelineDesc.blend.targets[0];
            targetDesc.blendEnabled     = true;
            targetDesc.dstColor         = LLGL::BlendOp::One;
            targetDesc.srcColor         = LLGL::BlendOp::One;
            targetDesc.colorArithmetic  = LLGL::BlendArithmetic::Subtract;
        }
        bundle[1].pipeline = renderer->CreatePipelineState(pipelineDesc);
    }

    static void PrintThreadsafe(std::mutex& mtx, const std::string& text)
    {
        std::lock_guard<std::mutex> guard { mtx };
        std::cout << text << std::endl;
    }

    static void EncodeSecondaryCommandBuffer(
        Bundle&         bundle,
        std::uint32_t   numIndices,
        std::mutex&     mtx,
        std::string     threadName)
    {
        // Print thread start
        PrintThreadsafe(mtx, "Enter thread: " + threadName);

        // Encode command buffer
        auto& cmdBuffer = *bundle.secondaryCmdBuffer;

        cmdBuffer.Begin();
        {
            cmdBuffer.SetPipelineState(*bundle.pipeline);
            cmdBuffer.SetResourceHeap(*bundle.resourceHeap);
            cmdBuffer.DrawIndexed(numIndices, 0);
        }
        cmdBuffer.End();

        // Print thread end
        PrintThreadsafe(mtx, "Leave thread: " + threadName);
    }

    void EncodePrimaryCommandBuffer(LLGL::CommandBuffer& cmdBuffer, std::uint32_t swapBufferIndex, const std::string& threadName = "")
    {
        // Print thread start
        if (!threadName.empty())
            PrintThreadsafe(logMutex, "Enter thread: " + threadName);

        // Encode command buffer
        cmdBuffer.Begin();
        {
            // Set hardware buffers to draw the model
            cmdBuffer.SetVertexBuffer(*vertexBuffer);
            cmdBuffer.SetIndexBuffer(*indexBuffer);

            // Set the swap-chain as the initial render target
            cmdBuffer.BeginRenderPass(*swapChain, nullptr, 0, nullptr, swapBufferIndex);
            {
                // Clear color- and depth buffers, and set viewport
                cmdBuffer.Clear(LLGL::ClearFlags::ColorDepth, { backgroundColor });
                cmdBuffer.SetViewport(swapChain->GetResolution());

                // Draw scene with secondary command buffers
                #if ENABLE_SECONDARY_COMMAND_BUFFERS

                for (auto& bdl : bundle)
                    cmdBuffer.Execute(*bdl.secondaryCmdBuffer);

                #else // ENABLE_SECONDARY_COMMAND_BUFFERS

                for (auto& bdl : bundle)
                {
                    cmdBuffer.SetPipelineState(*bdl.pipeline);
                    cmdBuffer.SetResourceHeap(*bdl.resourceHeap);
                    cmdBuffer.DrawIndexed(numIndices, 0);
                }

                #endif // /ENABLE_SECONDARY_COMMAND_BUFFERS
            }
            cmdBuffer.EndRenderPass();
        }
        cmdBuffer.End();

        // Print thread end
        if (!threadName.empty())
            PrintThreadsafe(logMutex, "Leave thread: " + threadName);
    }

    void CreateCommandBuffers()
    {
        // Create primary command buffer
        LLGL::CommandBufferDescriptor cmdBufferDesc;
        {
            cmdBufferDesc.flags = LLGL::CommandBufferFlags::MultiSubmit;
        }
        for_range(i, swapChain->GetNumSwapBuffers())
            primaryCmdBuffer[i] = renderer->CreateCommandBuffer(cmdBufferDesc);

        #if ENABLE_SECONDARY_COMMAND_BUFFERS

        // Create secondary command buffers
        cmdBufferDesc.flags = (LLGL::CommandBufferFlags::Secondary | LLGL::CommandBufferFlags::MultiSubmit);

        // Start encoding secondary command buffers in parallel
        std::thread workerThread[2];

        for_range(i, 2)
        {
            // Create secondary command buffer
            bundle[i].secondaryCmdBuffer = renderer->CreateCommandBuffer(cmdBufferDesc);

            // Start worker thread to encode secondary command buffer
            workerThread[i] = std::thread(
                Example_MultiThreading::EncodeSecondaryCommandBuffer,
                std::ref(bundle[i]),
                numIndices,
                std::ref(logMutex),
                "workerThread[" + std::to_string(i) + "]"
            );
        }

        #endif // /ENABLE_SECONDARY_COMMAND_BUFFERS

        // Encode primary command buffer
        for_range(i, swapChain->GetNumSwapBuffers())
            EncodePrimaryCommandBuffer(*primaryCmdBuffer[i], i, "mainThread");

        #if ENABLE_SECONDARY_COMMAND_BUFFERS

        // Wait for worker threads to finish
        for (auto& worker : workerThread)
        {
            if (worker.joinable())
                worker.join();
        }

        #endif // /ENABLE_SECONDARY_COMMAND_BUFFERS
    }

    void Transform(Bundle::Matrices& matrices, const Gs::Vector3f& pos, const Gs::Vector3f& axis, float angle)
    {
        matrices.wMatrix.LoadIdentity();
        Gs::Translate(matrices.wMatrix, pos);
        Gs::RotateFree(matrices.wMatrix, axis.Normalized(), angle);
        matrices.wvpMatrix = projection * matrices.wMatrix;
    }

    void UpdateScene()
    {
        if (input.KeyDown(LLGL::Key::Tab))
        {
            std::thread workerThread[2];

            for_range(i, 2)
            {
                // Start worker thread to encode secondary command buffer
                workerThread[i] = std::thread(
                    Example_MultiThreading::EncodeSecondaryCommandBuffer,
                    std::ref(bundle[i]),
                    numIndices,
                    std::ref(logMutex),
                    "workerThread[" + std::to_string(i) + "]"
                );
            }

            // Wait for worker threads to finish
            for (auto& worker : workerThread)
            {
                if (worker.joinable())
                    worker.join();
            }

        }

        // Animate rotation
        static float rotation;
        rotation += 0.01f;

        // Update scene matrices
        Transform(bundle[0].matrices, { -1, 0, 8 }, { +1, 1, 1 }, -rotation);
        Transform(bundle[1].matrices, { +1, 0, 8 }, { -1, 1, 1 }, +rotation);

        // Update constant buffer
        for (auto& bdl : bundle)
        {
            renderer->WriteBuffer(
                *bdl.constantBuffer,
                0,
                &(bdl.matrices),
                sizeof(bdl.matrices)
            );
        }
    }

    void DrawScene()
    {
        // Submit primary command buffer and present result
        measure.Start();
        commandQueue->Submit(*primaryCmdBuffer[swapChain->GetCurrentSwapIndex()]);
        measure.Stop();
    }

    void OnResize(const LLGL::Extent2D& /*resoluion*/) override
    {
        // Encode primary command buffer again to update resolution in constant buffer
        for_range(i, swapChain->GetNumSwapBuffers())
            EncodePrimaryCommandBuffer(*primaryCmdBuffer[i], i);
    }

    void OnDrawFrame() override
    {
        UpdateScene();
        DrawScene();
    }

};

LLGL_IMPLEMENT_EXAMPLE(Example_MultiThreading);

[gdianaty]

Just a quick note -- if I also update the main cmdbuf(s) (non-secondary), the bug does not occur in my toy engine codebase where this bug was discovered.

[LukasBanana]

Secondary command buffers that are executed via Execute on a primary command buffer must not be updated unless all primary command buffers on which such secondary command buffers were used are also updated. I shall add that information to the docs as I can see that this is not obvious. I fell for a similar pitfall when I had to update the interface to allow specifying a specific swap-chain buffer for the BeginRenderPass command, since you cannot expect the underlying API to use the "current buffer". Similarly, you cannot expect the API to use the "current state" of a command buffer when submitting it via Execute.

[gdianaty]

Thanks so much for the clarification and swift answer.

Keep up the stellar work!

[gdianaty]

Sorry to resurrect this thread -- I'm currently applying your suggestion and it's still indefinitely hanging.

Here's my updated UpdateScene() function from the above example, but tweaked to encode the primary command buffer as well. Again this is a reproducible example based on something from my larger codebase.

    void UpdateScene()
    {
        if (input.KeyDown(LLGL::Key::Tab))
        {
            std::thread workerThread[2];

            for_range(i, 2)
            {
                // Start worker thread to encode secondary command buffer
                workerThread[i] = std::thread(
                    Example_MultiThreading::EncodeSecondaryCommandBuffer,
                    std::ref(bundle[i]),
                    numIndices,
                    std::ref(logMutex),
                    "workerThread[" + std::to_string(i) + "]"
                );
            }

            // Wait for worker threads to finish
            for (auto& worker : workerThread)
            {
                if (worker.joinable())
                    worker.join();
            }

            for_range(i, swapChain->GetNumSwapBuffers())
                EncodePrimaryCommandBuffer(*primaryCmdBuffer[i], i, "mainThread");
        }

        // Animate rotation
        static float rotation;
        rotation += 0.01f;

        // Update scene matrices
        Transform(bundle[0].matrices, { -1, 0, 8 }, { +1, 1, 1 }, -rotation);
        Transform(bundle[1].matrices, { +1, 0, 8 }, { -1, 1, 1 }, +rotation);

        // Update constant buffer
        for (auto& bdl : bundle)
        {
            renderer->WriteBuffer(
                *bdl.constantBuffer,
                0,
                &(bdl.matrices),
                sizeof(bdl.matrices)
            );
        }
    }

[gdianaty]

I should note that order of update doesn't matter. If we do primary first and then secondary, the bug persists.

[LukasBanana]

Thanks for the update. I reproduced the issue and this should be fixed now with dd5c9e4. This will signal the fence for any command buffer that hasn't been submitted explicitly to the command queue, i.e. all secondary command buffers when they are re-encoded and also primary command buffers if they are redundantly re-encoded before ever being submitted to the command queue. Feel free to re-open anytime if this still doesn't work for you.

[gdianaty]

Gotchya! Testing now, I'll report back when I have results.

Thanks for the fix 🙏

[gdianaty]

So -- good news and bad news.

The good news is that the example as described above no longer reproduces the bug.

The bad news is that the bug still occurs in my codebase -- albeit after a few more frames have elapsed.

After some further testing, it appears that the following version of Example_MultiThreading still causes the bug. The only difference is that I have commented out the if (input.KeyDown(LLGL::Key::Tab)) expression, causing it to re-enocde all command buffers during every call to UpdateScene().

Please let it be known for the record that I discovered this by grabbing the edge of the window and moving it around while pressing tab repeatedly. Truly an excellent example of Looney Tunes debugging.

From the looks of things, it seems that the bug only happens if you're frantically updating the command buffer(s) for whatever reason. In the case of my codebase, I have a switch to enable all commandbuffers to update as often as possible.

/*
 * Example.cpp (Example_MultiThreading)
 *
 * Copyright (c) 2015 Lukas Hermanns. All rights reserved.
 * Licensed under the terms of the BSD 3-Clause license (see LICENSE.txt).
 */

#include <ExampleBase.h>
#include <LLGL/Utils/ForRange.h>
#include <thread>
#include <mutex>
#include <chrono>
#include <iomanip>


 // Enables/disables the use of two secondary command buffers
#define ENABLE_SECONDARY_COMMAND_BUFFERS 1

class Measure
{

    using Clock = std::chrono::system_clock;
    using TimePoint = std::chrono::time_point<Clock>;
    using Ticks = std::chrono::milliseconds;

public:

    // Interval (in milliseconds) to the next measurement result.
    Measure(std::uint64_t interval = 1000, const std::string& title = "Average Time") :
        interval_{ interval },
        title_{ title }
    {
    }

    void Start()
    {
        // Start timer
        timer_.Start();
    }

    void Stop()
    {
        // Take sample
        elapsed_ += timer_.Stop();
        ++samples_;

        // Check if average elapsed time can be printed again
        auto end = Clock::now();
        auto diff = std::chrono::duration_cast<Ticks>(end - intervalStartTime_);

        if (diff.count() >= static_cast<long long>(interval_))
        {
            Print();
            intervalStartTime_ = Clock::now();
        }
    }

private:

    void Print()
    {
        if (samples_ > 0)
        {
            auto averageTime = static_cast<double>(elapsed_);
            averageTime /= static_cast<double>(timer_.GetFrequency());
            averageTime *= 1000000.0;
            averageTime /= static_cast<double>(samples_);

            std::cout << title_ << ": ";
            std::cout << std::fixed << std::setprecision(6) << averageTime << " microseconds";
            std::cout << "         \r";
            std::flush(std::cout);

            samples_ = 0;
            elapsed_ = 0;
        }
    }

private:

    Stopwatch       timer_;
    std::uint64_t   interval_ = 0;
    TimePoint       intervalStartTime_;
    std::uint64_t   samples_ = 0;
    std::uint64_t   elapsed_ = 0;
    std::string     title_;

};

constexpr std::uint32_t maxNumSwapBuffers = 3;

class Example_MultiThreading : public ExampleBase
{

    ShaderPipeline              shaderPipeline;
    LLGL::Buffer* vertexBuffer = nullptr;
    LLGL::Buffer* indexBuffer = nullptr;
    LLGL::PipelineLayout* pipelineLayout = nullptr;
    LLGL::CommandBuffer* primaryCmdBuffer[maxNumSwapBuffers] = {};

    std::uint32_t               numIndices = 0;
    std::mutex                  logMutex;

    Measure                     measure;

    struct Bundle
    {
        LLGL::PipelineState* pipeline = nullptr;
        LLGL::Buffer* constantBuffer = nullptr;
        LLGL::ResourceHeap* resourceHeap = nullptr;
        LLGL::CommandBuffer* secondaryCmdBuffer = nullptr;

        struct Matrices
        {
            Gs::Matrix4f        wvpMatrix;
            Gs::Matrix4f        wMatrix;
        }
        matrices;
    }
    bundle[2];

public:

    Example_MultiThreading() :
        ExampleBase{ "LLGL Example: MultiThreading" }
    {
        auto vertexFormat = CreateBuffers();
        LoadShaders(vertexFormat);
        CreatePipelines();
        CreateCommandBuffers();
    }

private:

    LLGL::VertexFormat CreateBuffers()
    {
        // Specify vertex format
        LLGL::VertexFormat vertexFormat;
        vertexFormat.AppendAttribute({ "position", LLGL::Format::RGB32Float });
        vertexFormat.AppendAttribute({ "normal",   LLGL::Format::RGB32Float });
        vertexFormat.AppendAttribute({ "texCoord", LLGL::Format::RG32Float });

        // Generate data for mesh buffers
        auto indices = GenerateTexturedCubeTriangleIndices();
        auto vertices = GenerateTexturedCubeVertices();
        numIndices = static_cast<std::uint32_t>(indices.size());

        // Create buffers for a simple 3D cube model
        vertexBuffer = CreateVertexBuffer(vertices, vertexFormat);
        indexBuffer = CreateIndexBuffer(indices, LLGL::Format::R32UInt);

        for (auto& bdl : bundle)
            bdl.constantBuffer = CreateConstantBuffer(bdl.matrices);

        return vertexFormat;
    }

    void LoadShaders(const LLGL::VertexFormat& vertexFormat)
    {
        // Load shader program
        shaderPipeline = LoadStandardShaderPipeline({ vertexFormat });
    }

    void CreatePipelines()
    {
        // Create pipeline layout
        pipelineLayout = renderer->CreatePipelineLayout(LLGL::Parse("heap{cbuffer(Scene@1):vert}"));

        // Create resource view heap
        for (auto& bdl : bundle)
            bdl.resourceHeap = renderer->CreateResourceHeap(pipelineLayout, { bdl.constantBuffer });

        // Setup graphics pipeline descriptors
        LLGL::GraphicsPipelineDescriptor pipelineDesc;
        {
            // Set references to shader program, and pipeline layout
            pipelineDesc.vertexShader = shaderPipeline.vs;
            pipelineDesc.fragmentShader = shaderPipeline.ps;
            pipelineDesc.pipelineLayout = pipelineLayout;
            pipelineDesc.rasterizer.multiSampleEnabled = (GetSampleCount() > 1);

            // Enable depth test and writing
            pipelineDesc.depth.testEnabled = true;
            pipelineDesc.depth.writeEnabled = true;

            // Enable back-face culling
            pipelineDesc.rasterizer.cullMode = LLGL::CullMode::Back;
        }

        // Create first graphics pipeline
        bundle[0].pipeline = renderer->CreatePipelineState(pipelineDesc);

        // Create second graphics pipeline
        {
            auto& targetDesc = pipelineDesc.blend.targets[0];
            targetDesc.blendEnabled = true;
            targetDesc.dstColor = LLGL::BlendOp::One;
            targetDesc.srcColor = LLGL::BlendOp::One;
            targetDesc.colorArithmetic = LLGL::BlendArithmetic::Subtract;
        }
        bundle[1].pipeline = renderer->CreatePipelineState(pipelineDesc);
    }

    static void PrintThreadsafe(std::mutex& mtx, const std::string& text)
    {
        std::lock_guard<std::mutex> guard{ mtx };
        std::cout << text << std::endl;
    }

    static void EncodeSecondaryCommandBuffer(
        Bundle& bundle,
        std::uint32_t   numIndices,
        std::mutex& mtx,
        std::string     threadName)
    {
        // Print thread start
        PrintThreadsafe(mtx, "Enter thread: " + threadName);

        // Encode command buffer
        auto& cmdBuffer = *bundle.secondaryCmdBuffer;

        cmdBuffer.Begin();
        {
            cmdBuffer.SetPipelineState(*bundle.pipeline);
            cmdBuffer.SetResourceHeap(*bundle.resourceHeap);
            cmdBuffer.DrawIndexed(numIndices, 0);
        }
        cmdBuffer.End();

        // Print thread end
        PrintThreadsafe(mtx, "Leave thread: " + threadName);
    }

    void EncodePrimaryCommandBuffer(LLGL::CommandBuffer& cmdBuffer, std::uint32_t swapBufferIndex, const std::string& threadName = "")
    {
        // Print thread start
        if (!threadName.empty())
            PrintThreadsafe(logMutex, "Enter thread: " + threadName);

        // Encode command buffer
        cmdBuffer.Begin();
        {
            // Set hardware buffers to draw the model
            cmdBuffer.SetVertexBuffer(*vertexBuffer);
            cmdBuffer.SetIndexBuffer(*indexBuffer);

            // Set the swap-chain as the initial render target
            cmdBuffer.BeginRenderPass(*swapChain, nullptr, 0, nullptr, swapBufferIndex);
            {
                // Clear color- and depth buffers, and set viewport
                cmdBuffer.Clear(LLGL::ClearFlags::ColorDepth, { backgroundColor });
                cmdBuffer.SetViewport(swapChain->GetResolution());

                // Draw scene with secondary command buffers
#if ENABLE_SECONDARY_COMMAND_BUFFERS

                for (auto& bdl : bundle)
                    cmdBuffer.Execute(*bdl.secondaryCmdBuffer);

#else // ENABLE_SECONDARY_COMMAND_BUFFERS

                for (auto& bdl : bundle)
                {
                    cmdBuffer.SetPipelineState(*bdl.pipeline);
                    cmdBuffer.SetResourceHeap(*bdl.resourceHeap);
                    cmdBuffer.DrawIndexed(numIndices, 0);
                }

#endif // /ENABLE_SECONDARY_COMMAND_BUFFERS
            }
            cmdBuffer.EndRenderPass();
        }
        cmdBuffer.End();

        // Print thread end
        if (!threadName.empty())
            PrintThreadsafe(logMutex, "Leave thread: " + threadName);
    }

    void CreateCommandBuffers()
    {
        // Create primary command buffer
        LLGL::CommandBufferDescriptor cmdBufferDesc;
        {
            cmdBufferDesc.flags = LLGL::CommandBufferFlags::MultiSubmit;
        }
        for_range(i, swapChain->GetNumSwapBuffers())
            primaryCmdBuffer[i] = renderer->CreateCommandBuffer(cmdBufferDesc);

#if ENABLE_SECONDARY_COMMAND_BUFFERS

        // Create secondary command buffers
        cmdBufferDesc.flags = (LLGL::CommandBufferFlags::Secondary | LLGL::CommandBufferFlags::MultiSubmit);

        // Start encoding secondary command buffers in parallel
        std::thread workerThread[2];

        for_range(i, 2)
        {
            // Create secondary command buffer
            bundle[i].secondaryCmdBuffer = renderer->CreateCommandBuffer(cmdBufferDesc);

            // Start worker thread to encode secondary command buffer
            workerThread[i] = std::thread(
                Example_MultiThreading::EncodeSecondaryCommandBuffer,
                std::ref(bundle[i]),
                numIndices,
                std::ref(logMutex),
                "workerThread[" + std::to_string(i) + "]"
            );
        }

#endif // /ENABLE_SECONDARY_COMMAND_BUFFERS

        // Encode primary command buffer
        for_range(i, swapChain->GetNumSwapBuffers())
            EncodePrimaryCommandBuffer(*primaryCmdBuffer[i], i, "mainThread");

#if ENABLE_SECONDARY_COMMAND_BUFFERS

        // Wait for worker threads to finish
        for (auto& worker : workerThread)
        {
            if (worker.joinable())
                worker.join();
        }

#endif // /ENABLE_SECONDARY_COMMAND_BUFFERS
    }

    void Transform(Bundle::Matrices& matrices, const Gs::Vector3f& pos, const Gs::Vector3f& axis, float angle)
    {
        matrices.wMatrix.LoadIdentity();
        Gs::Translate(matrices.wMatrix, pos);
        Gs::RotateFree(matrices.wMatrix, axis.Normalized(), angle);
        matrices.wvpMatrix = projection * matrices.wMatrix;
    }


    void UpdateScene()
    {
        //if (input.KeyDown(LLGL::Key::Tab))
        {
            std::thread workerThread[2];

            for_range(i, 2)
            {
                // Start worker thread to encode secondary command buffer
                workerThread[i] = std::thread(
                    Example_MultiThreading::EncodeSecondaryCommandBuffer,
                    std::ref(bundle[i]),
                    numIndices,
                    std::ref(logMutex),
                    "workerThread[" + std::to_string(i) + "]"
                );
            }

            // Wait for worker threads to finish
            for (auto& worker : workerThread)
            {
                if (worker.joinable())
                    worker.join();
            }

            for_range(i, swapChain->GetNumSwapBuffers())
                EncodePrimaryCommandBuffer(*primaryCmdBuffer[i], i, "mainThread");
        }

        // Animate rotation
        static float rotation;
        rotation += 0.01f;

        // Update scene matrices
        Transform(bundle[0].matrices, { -1, 0, 8 }, { +1, 1, 1 }, -rotation);
        Transform(bundle[1].matrices, { +1, 0, 8 }, { -1, 1, 1 }, +rotation);

        // Update constant buffer
        for (auto& bdl : bundle)
        {
            renderer->WriteBuffer(
                *bdl.constantBuffer,
                0,
                &(bdl.matrices),
                sizeof(bdl.matrices)
            );
        }
    }


    void DrawScene()
    {
        // Submit primary command buffer and present result
        measure.Start();
        commandQueue->Submit(*primaryCmdBuffer[swapChain->GetCurrentSwapIndex()]);
        measure.Stop();
    }

    void OnResize(const LLGL::Extent2D& /*resoluion*/) override
    {
        // Encode primary command buffer again to update resolution in constant buffer
        for_range(i, swapChain->GetNumSwapBuffers())
            EncodePrimaryCommandBuffer(*primaryCmdBuffer[i], i);
    }

    void OnDrawFrame() override
    {
        UpdateScene();
        DrawScene();
    }

};

LLGL_IMPLEMENT_EXAMPLE(Example_MultiThreading);

[gdianaty]

(P.S. I lack the ability to reopen this issue, otherwise I would. Sorry!)