7186009: [macOS] No way to hide(), dispose() FileDialog

src/java.desktop/macosx/classes/sun/lwawt/macosx/CFileDialog.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -109,7 +109,10 @@ public void run() {
                 accessor.setFiles(target, files);
             } finally {
                 // Java2 Dialog waits for hide to let show() return
-                target.dispose();
+                AWTAccessor.FileDialogAccessor accessor = AWTAccessor.getFileDialogAccessor();
+                if (!accessor.isBeingDisposed(target)) {
+                    target.dispose();
+                }
             }
         }
     }
@@ -121,12 +124,15 @@ public void run() {
         this.target = target;
     }
 
+    private volatile long nativeCFileDialogPtr;
+    private volatile long nativeWindowID;
+
+    private native void nativeDispose();
+
     @Override
     public void dispose() {
         LWCToolkit.targetDisposedPeer(target, this);
-        // Unlike other peers, we do not have a native model pointer to
-        // dispose of because the save and open panels are never released by
-        // an application.
+        setVisible(false);
     }
 
     @Override
@@ -135,9 +141,13 @@ public void setVisible(boolean visible) {
             // Java2 Dialog class requires peer to run code in a separate thread
             // and handles keeping the call modal
             new Thread(null, new Task(), "FileDialog", 0, false).start();
+        } else {
+             // This call doesn't directly dispose the dialog, but if it is being
+             // displayed it stops the modal loop which hides it and returns control
+            if ((nativeCFileDialogPtr != 0L) && (nativeWindowID != 0L)) {
+                nativeDispose();
+            }
         }
-        // We hide ourself before "show" returns - setVisible(false)
-        // doesn't apply
     }
 
     /**

src/java.desktop/macosx/native/libawt_lwawt/awt/CFileDialog.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -42,6 +42,9 @@
     NSString *fDirectory;
     NSString *fFile;
 
+    @public
+    BOOL inModalLoop;
+
     // File dialog's mode
     jint fMode;
 

src/java.desktop/macosx/native/libawt_lwawt/awt/CFileDialog.m

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -60,6 +60,7 @@ - (id)initWithFilter:(jboolean)inHasFilter
         fNavigateApps = inNavigateApps;
         fChooseDirectories = inChooseDirectories;
         fPanelResult = NSCancelButton;
+        inModalLoop = NO;
     }
 
     return self;
@@ -107,6 +108,11 @@ - (void)safeSaveOrLoad {
         thePanel = [NSOpenPanel openPanel];
     }
 
+    JNIEnv *env = [ThreadUtilities getJNIEnv];
+    DECLARE_CLASS(jc_CFileDialog, "sun/lwawt/macosx/CFileDialog");
+    DECLARE_FIELD(jc_windowID, jc_CFileDialog, "nativeWindowID", "J");
+    (*env)->SetLongField(env, fFileDialog, jc_windowID, ptr_to_jlong(thePanel));
+
     if (thePanel != nil) {
         [thePanel setTitle:fTitle];
 
@@ -123,7 +129,9 @@ - (void)safeSaveOrLoad {
         }
 
         [thePanel setDelegate:self];
+        inModalLoop = YES;
         fPanelResult = [thePanel runModalForDirectory:fDirectory file:fFile];
+        inModalLoop = NO;
         [thePanel setDelegate:nil];
 
         if ([self userClickedOK]) {
@@ -137,6 +145,7 @@ - (void)safeSaveOrLoad {
         }
     }
 
+    (*env)->SetLongField(env, fFileDialog, jc_windowID, ptr_to_jlong(0L));
     [self disposer];
 }
 
@@ -183,6 +192,34 @@ - (NSArray *)URLs {
 }
 @end
 
+JNIEXPORT void JNICALL Java_sun_lwawt_macosx_CFileDialog_nativeDispose(JNIEnv *env, jobject peer) {
+
+    DECLARE_CLASS(jc_CFileDialog, "sun/lwawt/macosx/CFileDialog");
+
+    DECLARE_FIELD(jc_nativePtr, jc_CFileDialog, "nativeCFileDialogPtr", "J");
+    CFileDialog *dialogDelegate = (CFileDialog*)jlong_to_ptr((*env)->GetLongField(env, peer, jc_nativePtr));
+
+    if (dialogDelegate == nil) {
+        return;
+    }
+
+    DECLARE_FIELD(jc_windowID, jc_CFileDialog, "nativeWindowID", "J");
+    NSWindow *windowID = (NSWindow*)jlong_to_ptr((*env)->GetLongField(env, peer, jc_windowID));
+    if (windowID == nil) {
+        return;
+    }
+
+    [ThreadUtilities performOnMainThreadWaiting:[NSThread isMainThread] block:^(){
+        if (dialogDelegate->inModalLoop == YES) {
+            NSApplication *app = [NSApplication sharedApplication];
+            NSWindow *modalWindow = [app modalWindow];
+            if (modalWindow != nil && (modalWindow == windowID)) {
+                [app stopModalWithCode:NSModalResponseCancel];
+            }
+        }
+    }];
+   }
+
 /*
  * Class:     sun_lwawt_macosx_CFileDialog
  * Method:    nativeRunFileDialog
@@ -214,6 +251,11 @@ - (NSArray *)URLs {
                                                  canChooseDirectories:chooseDirectories
                                                               withEnv:env];
 
+    DECLARE_CLASS_RETURN(jc_CFileDialog, "sun/lwawt/macosx/CFileDialog", NULL);
+
+    DECLARE_FIELD_RETURN(jc_nativePtr, jc_CFileDialog, "nativeCFileDialogPtr", "J", NULL);
+    (*env)->SetLongField(env, peer, jc_nativePtr, ptr_to_jlong(dialogDelegate));
+
     [ThreadUtilities performOnMainThread:@selector(safeSaveOrLoad)
                                  on:dialogDelegate
                          withObject:nil
@@ -233,6 +275,7 @@ - (NSArray *)URLs {
         }];
     }
 
+    (*env)->SetLongField(env, peer, jc_nativePtr, ptr_to_jlong(0L));
     [dialogDelegate release];
 JNI_COCOA_EXIT(env);
     return returnValue;

src/java.desktop/share/classes/java/awt/FileDialog.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1995, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1995, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -160,6 +160,9 @@ public boolean isMultipleMode(FileDialog fileDialog) {
                         return fileDialog.multipleMode;
                     }
                 }
+                public boolean isBeingDisposed(FileDialog fileDialog) {
+                    return fileDialog.isInDispose;
+                }
             });
     }
 

src/java.desktop/share/classes/sun/awt/AWTAccessor.java

@@ -545,6 +545,11 @@ public interface FileDialogAccessor {
          * Returns whether the file dialog allows the multiple file selection.
          */
         boolean isMultipleMode(FileDialog fileDialog);
+
+        /*
+         * Returns whether dispose is being run
+         */
+        boolean isBeingDisposed(FileDialog fileDialog);
     }
 
     /*

test/jdk/ProblemList.txt

@@ -247,32 +247,6 @@ java/awt/Clipboard/ImageTransferTest.java 8030710 generic-all
 java/awt/Clipboard/NoDataConversionFailureTest.java 8234140 macosx-all
 java/awt/Dialog/ModalExcludedTest.java 7125054 macosx-all
 java/awt/Frame/MiscUndecorated/RepaintTest.java 8266244 macosx-aarch64
-java/awt/Modal/FileDialog/FileDialogAppModal1Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogAppModal2Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogAppModal3Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogAppModal4Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogAppModal5Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogAppModal6Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogDocModal1Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogDocModal2Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogDocModal3Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogDocModal4Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogDocModal5Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogDocModal6Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogDocModal7Test.java 7186009 macosx-all,linux-all
-java/awt/Modal/FileDialog/FileDialogModal1Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogModal2Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogModal3Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogModal4Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogModal5Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogModal6Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogNonModal1Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogNonModal2Test.java 7186009 macosx-all,linux-all
-java/awt/Modal/FileDialog/FileDialogNonModal3Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogNonModal4Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogNonModal5Test.java 7186009 macosx-all
-java/awt/Modal/FileDialog/FileDialogNonModal6Test.java 7186009 macosx-all,linux-all
-java/awt/Modal/FileDialog/FileDialogNonModal7Test.java 7186009 macosx-all,linux-all
 java/awt/Modal/FileDialog/FileDialogTKModal1Test.java 8196430 generic-all
 java/awt/Modal/FileDialog/FileDialogTKModal2Test.java 8196430 generic-all
 java/awt/Modal/FileDialog/FileDialogTKModal3Test.java 8196430 generic-all

test/jdk/java/awt/Modal/FileDialog/FileDialogDocModal7Test.java

@@ -37,6 +37,7 @@
  * @build TestDialog
  * @build TestFrame
  * @build TestWindow
+ * @requires (os.family != "mac")
  * @run main FileDialogDocModal7Test
  */
 

test/jdk/java/awt/Modal/FileDialog/FileDialogNonModal7Test.java

@@ -37,6 +37,7 @@
  * @build TestDialog
  * @build TestFrame
  * @build TestWindow
+ * @requires (os.family != "mac")
  * @run main FileDialogNonModal7Test
  */