Fix process hanging upon exit. Fix re-hooking Direct-X when Device is Lost or Reset.

Author: Brandon-T

RemoteInput/Platform/NativeHooks_Windows.cxx

@@ -46,7 +46,6 @@ std::unique_ptr<Hook> directx_device9_endscene_hook{nullptr};
 std::unique_ptr<Hook> directx_device9_reset_hook{nullptr};
 std::unique_ptr<Hook> directx_device11_swapchain_present_hook{nullptr};
 
-bool is_dx_hooked = false;
 IDirect3DDevice9* current_dx_device = nullptr;
 void HookD3D9Device(IDirect3DDevice9* pDevice, bool force = false) noexcept;
 
@@ -615,7 +614,6 @@ HRESULT __cdecl JavaDirectXCopyImageToIntArgbSurface(IDirect3DSurface9 *pSurface
     {
         HookD3D9Device(pDevice);
         pDevice->Release();
-        is_dx_hooked = true;
     }
 
     return directx_xrgb_hook->call<HRESULT, decltype(JavaDirectXCopyImageToIntArgbSurface)>(pSurface, pDstInfo, srcx, srcy, srcWidth, srcHeight, dstx, dsty);
@@ -649,7 +647,6 @@ HRESULT __cdecl JavaDirectXCopyImageToIntXrgbSurface(SurfaceDataRasInfo *pSrcInf
     {
         HookD3D9Device(pDevice);
         pDevice->Release();
-        is_dx_hooked = true;
     }
 
     return directx_xrgb_hook->call<HRESULT, decltype(JavaDirectXCopyImageToIntXrgbSurface)>(pSrcInfo, srctype, pDstSurfaceRes, srcx, srcy, srcWidth, srcHeight, dstx, dsty);
@@ -1001,10 +998,7 @@ HRESULT __stdcall D3D9_CreateDevice(IDirect3D9* pD3D, UINT Adapter, D3DDEVTYPE D
         HRESULT result = directx_d3d9_createdevice_hook->call<HRESULT, decltype(D3D9_CreateDevice)>(pD3D, Adapter, DeviceType, hFocusWindow, BehaviorFlags, pPresentationParameters, ppReturnedDeviceInterface);
         if (ppReturnedDeviceInterface && *ppReturnedDeviceInterface)
         {
-            if (!is_dx_hooked)
-            {
-                HookD3D9Device(*ppReturnedDeviceInterface);
-            }
+            HookD3D9Device(*ppReturnedDeviceInterface);
         }
         return result;
     }
@@ -1016,83 +1010,83 @@ HRESULT __stdcall D3D9Device_EndScene(IDirect3DDevice9* device) noexcept
 {
     extern std::unique_ptr<ControlCenter> control_center;
 
-    if (directx_device9_endscene_hook)
+    if (control_center)
     {
-        if (control_center)
+        D3DVIEWPORT9 viewport;
+        if (SUCCEEDED(device->GetViewport(&viewport)))
         {
-            D3DVIEWPORT9 viewport;
-            if (SUCCEEDED(device->GetViewport(&viewport)))
-            {
-                std::int32_t width = static_cast<std::int32_t>(viewport.Width);
-                std::int32_t height = static_cast<std::int32_t>(viewport.Height);
+            std::int32_t width = static_cast<std::int32_t>(viewport.Width);
+            std::int32_t height = static_cast<std::int32_t>(viewport.Height);
 
-                std::int32_t x = 0;
-                std::int32_t y = 0;
-                std::size_t applet_width = 0;
-                std::size_t applet_height = 0;
+            std::int32_t x = 0;
+            std::int32_t y = 0;
+            std::size_t applet_width = 0;
+            std::size_t applet_height = 0;
 
-                control_center->get_applet_dimensions(&x, &y, &applet_width, &applet_height);
-                control_center->set_target_dimensions(static_cast<std::int32_t>(applet_width), static_cast<std::int32_t>(applet_height));
+            control_center->get_applet_dimensions(&x, &y, &applet_width, &applet_height);
+            control_center->set_target_dimensions(static_cast<std::int32_t>(applet_width), static_cast<std::int32_t>(applet_height));
 
-                /*if (width != applet_width || height != applet_height)
-                {
-                    // Possibly menu open, render normally
-                    return directx_device9_endscene_hook->call<HRESULT, decltype(D3D9Device_EndScene)>(device);
-                }*/
-
-                control_center->set_target_dimensions(static_cast<std::int32_t>(width), static_cast<std::int32_t>(height));
-
-                bool minimized = false;
-                ImageFormat image_format = control_center->get_image_format();
-                dx_read_pixels(device, control_center->get_image(), width, height, minimized, image_format);
-
-                IDirect3DStateBlock9* block;
-                device->CreateStateBlock(D3DSBT_ALL, &block);
-                block->Capture();
-
-                device->SetRenderState(D3DRS_LIGHTING, FALSE);
-                device->SetRenderState(D3DRS_FOGENABLE, FALSE);
-                device->SetRenderState(D3DRS_ZENABLE, D3DZB_FALSE);
-                device->SetRenderState(D3DRS_CULLMODE, D3DCULL_NONE);
-                device->SetRenderState(D3DRS_ALPHABLENDENABLE, TRUE); //DISABLED
-                device->SetRenderState(D3DRS_BLENDOP, D3DBLENDOP_ADD);
-                device->SetRenderState(D3DRS_SRCBLEND, D3DBLEND_SRCALPHA);
-                device->SetRenderState(D3DRS_DESTBLEND, D3DBLEND_INVSRCALPHA);
-
-                if (control_center->get_debug_graphics() && !minimized)
-                {
-                    static IDirect3DTexture9* texture = nullptr;
-                    dx_load_texture(device, texture, image_format, control_center->get_debug_image(), width, height);
+            /*if (width != applet_width || height != applet_height)
+            {
+                // Possibly menu open, render normally
+                return directx_device9_endscene_hook->call<HRESULT, decltype(D3D9Device_EndScene)>(device);
+            }*/
+
+            control_center->set_target_dimensions(static_cast<std::int32_t>(width), static_cast<std::int32_t>(height));
+
+            bool minimized = false;
+            ImageFormat image_format = control_center->get_image_format();
+            dx_read_pixels(device, control_center->get_image(), width, height, minimized, image_format);
+
+            IDirect3DStateBlock9* block;
+            device->CreateStateBlock(D3DSBT_ALL, &block);
+            block->Capture();
+
+            device->SetRenderState(D3DRS_LIGHTING, FALSE);
+            device->SetRenderState(D3DRS_FOGENABLE, FALSE);
+            device->SetRenderState(D3DRS_ZENABLE, D3DZB_FALSE);
+            device->SetRenderState(D3DRS_CULLMODE, D3DCULL_NONE);
+            device->SetRenderState(D3DRS_ALPHABLENDENABLE, TRUE); //DISABLED
+            device->SetRenderState(D3DRS_BLENDOP, D3DBLENDOP_ADD);
+            device->SetRenderState(D3DRS_SRCBLEND, D3DBLEND_SRCALPHA);
+            device->SetRenderState(D3DRS_DESTBLEND, D3DBLEND_INVSRCALPHA);
+
+            if (control_center->get_debug_graphics() && !minimized)
+            {
+                static IDirect3DTexture9* texture = nullptr;
+                dx_load_texture(device, texture, image_format, control_center->get_debug_image(), width, height);
 
-                    if (texture)
-                    {
-                        dx_draw_texture(device, texture, image_format, 0.0, 0.0, static_cast<float>(width), static_cast<float>(height));
-                    }
+                if (texture)
+                {
+                    dx_draw_texture(device, texture, image_format, 0.0, 0.0, static_cast<float>(width), static_cast<float>(height));
                 }
+            }
 
-                //Render Cursor
-                if (!minimized)
-                {
-                    x = -1;
-                    y = -1;
-                    control_center->get_applet_mouse_position(&x, &y);
+            //Render Cursor
+            if (!minimized)
+            {
+                x = -1;
+                y = -1;
+                control_center->get_applet_mouse_position(&x, &y);
 
-                    if (x > -1 && y > -1 && x <= width && y <= height)
-                    {
-                        device->SetTexture(0, nullptr);
-                        dx_draw_point(device, static_cast<float>(x), static_cast<float>(y), 2.5f, D3DCOLOR_RGBA(0xFF, 0x00, 0x00, 0xFF));
-                    }
+                if (x > -1 && y > -1 && x <= width && y <= height)
+                {
+                    device->SetTexture(0, nullptr);
+                    dx_draw_point(device, static_cast<float>(x), static_cast<float>(y), 2.5f, D3DCOLOR_RGBA(0xFF, 0x00, 0x00, 0xFF));
                 }
-
-                device->SetRenderState(D3DRS_ZFUNC,D3DCMP_NEVER);
-                device->SetTexture(0, nullptr);
-                device->SetPixelShader(nullptr);
-                device->SetVertexShader(nullptr);
-                block->Apply();
-                block->Release();
             }
+
+            device->SetRenderState(D3DRS_ZFUNC,D3DCMP_NEVER);
+            device->SetTexture(0, nullptr);
+            device->SetPixelShader(nullptr);
+            device->SetVertexShader(nullptr);
+            block->Apply();
+            block->Release();
         }
+    }
 
+    if (directx_device9_endscene_hook)
+    {
         return directx_device9_endscene_hook->call<HRESULT, decltype(D3D9Device_EndScene)>(device);
     }
 
@@ -1135,6 +1129,18 @@ void HookD3D9Device(IDirect3DDevice9* pDevice, bool force) noexcept
     auto* reset = reinterpret_cast<decltype(D3D9Device_Reset)*>(vTable[16]);
     directx_device9_reset_hook = std::make_unique<Hook>(reinterpret_cast<void*>(reset), reinterpret_cast<void*>(D3D9Device_Reset));
     directx_device9_reset_hook->apply();
+
+    if (!directx_device9_endscene_hook->is_enabled())
+    {
+        current_dx_device = nullptr;
+        directx_device9_endscene_hook.reset();
+    }
+
+    if (!directx_device9_reset_hook->is_enabled())
+    {
+        current_dx_device = nullptr;
+        directx_device9_reset_hook.reset();
+    }
 }
 #endif // defined
 

RemoteInput/Plugin/ControlCenter.cxx

@@ -118,7 +118,12 @@ ControlCenter::ControlCenter(std::int32_t pid, bool is_controller, std::unique_p
 
 ControlCenter::~ControlCenter()
 {
-    terminate();
+    //terminate();
+
+    if (io_controller)
+    {
+        io_controller.reset();
+    }
 
     if (sync_signal)
     {

RemoteInput/Plugin/InputOutput.cxx

@@ -97,7 +97,7 @@ static java::KeyEvent::KeyCodes control_keys_locations[] = {
     java::KeyEvent::KeyCodes::KEY_LOCATION_STANDARD
 };
 
-InputOutput::InputOutput(Reflection* reflection) noexcept : vm(nullptr), applet(reflection->getApplet()), mutex(), input_thread(2), event_queue(nullptr), currently_held_key(-1), held_keys(), x(-1), y(-1), w(-1), h(-1), click_count(0), keyboard_speed(0), keyboard_repeat_delay(0), mouse_buttons()
+InputOutput::InputOutput(Reflection* reflection) noexcept : vm(nullptr), applet(reflection->getApplet()), mutex(), input_thread(new ThreadPool(2)), event_queue(nullptr), currently_held_key(-1), held_keys(), x(-1), y(-1), w(-1), h(-1), click_count(0), keyboard_speed(0), keyboard_repeat_delay(0), mouse_buttons()
 {
     reflection->getEnv()->GetJavaVM(&vm);
 
@@ -158,7 +158,10 @@ InputOutput::InputOutput(Reflection* reflection) noexcept : vm(nullptr), applet(
 
 InputOutput::~InputOutput() noexcept
 {
-    this->input_thread.terminate();
+    // Purposely do NOT join the thread or reset! This will hang upon dll unload.
+    // We `release` it instead to prevent this.
+    //this->input_thread->terminate();
+    this->input_thread.release();
     this->event_queue.reset();
     this->vm = nullptr;
     this->applet = nullptr;
@@ -316,7 +319,7 @@ void InputOutput::hold_key(std::int32_t code) noexcept
 
                 if (this->keyboard_speed >= 0 && this->keyboard_repeat_delay >= 0)
                 {
-                    input_thread.add_task([&](std::atomic_bool &stopped) {
+                    input_thread->add_task([&](std::atomic_bool &stopped) {
                         if (this->keyboard_repeat_delay > 0)
                         {
                             yield_thread(std::chrono::milliseconds(this->keyboard_repeat_delay));

RemoteInput/Plugin/InputOutput.hxx

@@ -27,7 +27,7 @@ private:
     JavaVM* vm;
     jobject applet;
     std::mutex mutex;
-    ThreadPool input_thread;
+    std::unique_ptr<ThreadPool> input_thread;
     std::unique_ptr<java::RIEventQueue> event_queue;
 
     // MARK: - Input Variables