Reduce flicker when entering non-native full screen

src/MacVim/MMCoreTextView.m

@@ -1499,6 +1499,8 @@ - (NSRect)rectForRow:(int)row column:(int)col numRows:(int)nr
     if (col + nc == grid.cols) {
         const NSInteger insetRight = [[NSUserDefaults standardUserDefaults] integerForKey:MMTextInsetRightKey];
         CGFloat extraWidth = frame.size.width - insetRight - (rect.size.width + rect.origin.x);
+        if (extraWidth > cellSize.width * 4) // just a sane cap so Vim doesn't look really stretched when resized before Vim could catch up
+            extraWidth = cellSize.width * 4;
         rect.size.width += extraWidth;
     }
 

src/MacVim/MMWindow.m

@@ -213,11 +213,12 @@ - (IBAction)zoom:(id)sender
 
 - (IBAction)toggleFullScreen:(id)sender
 {
-    // HACK! This is an NSWindow method used to enter full-screen on OS X 10.7.
-    // We override it so that we can interrupt and pass this on to Vim first.
-    // An alternative hack would be to reroute the action message sent by the
-    // full-screen button in the top right corner of a window, but there could
-    // be other places where this action message is sent from.
+    // This is an NSWindow method used to enter full-screen since OS X 10.7
+    // Lion. We override it so that we can interrupt and pass this on to Vim
+    // first, as it is full-screen aware (":set fullscreen") and it's better to
+    // only have one path to enter full screen. For non-native full screen this
+    // does mean this button will now enter non-native full screen instead of
+    // native one.
     // To get to the original method (and enter Lion full-screen) we need to
     // call realToggleFullScreen: defined below.
 
@@ -227,11 +228,8 @@ - (IBAction)toggleFullScreen:(id)sender
 
 - (IBAction)realToggleFullScreen:(id)sender
 {
-#if (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7)
-    // HACK! See toggleFullScreen: comment above.
-    if ([NSWindow instancesRespondToSelector:@selector(toggleFullScreen:)])
-        [super toggleFullScreen:sender];
-#endif
+    // See toggleFullScreen: comment above.
+    [super toggleFullScreen:sender];
 }
 
 - (void)setToolbar:(NSToolbar *)toolbar

src/MacVim/MMWindowController.h

@@ -31,7 +31,9 @@
 
     BOOL                shouldResizeVimView; ///< Indicates there is a pending command to resize the Vim view
     BOOL                shouldKeepGUISize; ///< If on, the Vim view resize will try to fit in the existing window. If off, the window resizes to fit Vim view.
+
     BOOL                blockRenderUntilResize; ///< Indicates that there should be no text rendering until a Vim view resize is completed to avoid flicker.
+    NSRect              blockedRenderTextViewFrame; ///< The old screen-based coords for the text view when render was blocked.
 
     BOOL                shouldRestoreUserTopLeft;
     int                 updateToolbarFlag;

src/MacVim/MMWindowController.m

@@ -420,6 +420,7 @@ - (void)setTextDimensionsWithRows:(int)rows columns:(int)cols isLive:(BOOL)live
     vimView.pendingLiveResize = NO;
     if (blockRenderUntilResize) {
         blockRenderUntilResize = NO;
+        blockedRenderTextViewFrame = NSZeroRect;
         [vimView.textView setDrawRectOffset:NSZeroSize];
     }
     if (vimView.pendingLiveResizeQueued) {
@@ -506,6 +507,9 @@ - (void)resizeVimViewBlockRender
         shouldResizeVimView = YES;
         shouldKeepGUISize = YES;
         blockRenderUntilResize = YES;
+        blockedRenderTextViewFrame = [self.window convertRectToScreen:
+                                      [vimView convertRect:vimView.textView.frame
+                                                    toView:nil]];
         if (!vimController.isHandlingInputQueue)
             [self processInputQueueDidFinish];
     }
@@ -884,7 +888,6 @@ - (void)processInputQueueDidFinish
 
     const int oldTextViewRows = vimView.textView.pendingMaxRows;
     const int oldTextViewCols = vimView.textView.pendingMaxColumns;
-    const NSRect oldTextViewFrame = vimView.textView.frame;
     BOOL vimViewSizeChanged = NO;
 
     // NOTE: If the window has not been presented then we must avoid resizing
@@ -899,8 +902,6 @@ - (void)processInputQueueDidFinish
         // Setting 'guioptions+=k' will make shouldKeepGUISize true, which
         // means avoid resizing the window. Instead, resize the view instead
         // to keep the GUI window's size consistent.
-        // Note: Vim should always have requested shouldKeepGUISize to be true
-        //       when in full screen, but we check for it anyway for safety.
         bool avoidWindowResize = shouldKeepGUISize || fullScreenEnabled;
 
         if (!avoidWindowResize) {
@@ -939,16 +940,18 @@ - (void)processInputQueueDidFinish
 
     if (blockRenderUntilResize) {
         if (vimViewSizeChanged) {
-            const NSRect newTextViewFrame = vimView.textView.frame;
+            const NSRect newTextViewFrame = [self.window convertRectToScreen:[vimView convertRect:vimView.textView.frame toView:nil]];
 
             // We are currently blocking all rendering to prevent flicker. If
-            // the view frame moved (this happens if the tab or left scroll bar
-            // were shown/hidden) the user will see a temporary flicker as the
-            // text view was moved before Vim has udpated us with new draw calls
+            // the view frame moved (this happens if say the tab bar was shown
+            // or hidden) the user will see a temporary flicker as the text
+            // view was moved before Vim has updated us with new draw calls
             // to match the new size. To alleviate this, we temporarily apply
             // a drawing offset in the text view to counter the offset. To the
             // user it would appear that the text view hasn't moved at all.
-            [vimView.textView setDrawRectOffset:NSMakeSize(NSMinX(oldTextViewFrame) - NSMinX(newTextViewFrame), NSMaxY(oldTextViewFrame) - NSMaxY(newTextViewFrame))];
+            [vimView.textView setDrawRectOffset:
+             NSMakeSize(NSMinX(blockedRenderTextViewFrame) - NSMinX(newTextViewFrame),
+                        NSMaxY(blockedRenderTextViewFrame) - NSMaxY(newTextViewFrame))];
         } else {
             // We were blocking all rendering until Vim has been resized. However
             // in situations where we turned out to not need to resize Vim to
@@ -959,6 +962,7 @@ - (void)processInputQueueDidFinish
             // we need to resize) but turned out we set it to the same font so
             // the grid size is the same and no need to resize.
             blockRenderUntilResize = NO;
+            blockedRenderTextViewFrame = NSZeroRect;
             [vimView.textView setDrawRectOffset:NSZeroSize];
 
             [vimController sendMessage:RedrawMsgID data:nil];
@@ -1139,6 +1143,22 @@ - (void)enterFullScreen:(int)fuoptions backgroundColor:(NSColor *)back
         // custom full-screen can appear on any screen, as opposed to native
         // full-screen which always uses the main screen.)
         if (windowPresented) {
+            const BOOL shouldPreventFlicker = (fuoptions & FUOPT_MAXVERT) && (fuoptions & FUOPT_MAXHORZ);
+            if (shouldPreventFlicker) {
+                // Prevent visual flickering by temporarily blocking new render
+                // until Vim has updated/resized itself.
+                // We don't do the same when exiting full screen because when
+                // going in this direction the flickering is less noticeable
+                // and it looks odd when the user sees a clamped view.
+                // Also, don't do this if maxvert/maxhorz not set because it
+                // looks quite off in that situation as Vim is supposed to move
+                // visually.
+                blockRenderUntilResize = YES;
+                blockedRenderTextViewFrame = [decoratedWindow convertRectToScreen:
+                                              [vimView convertRect:vimView.textView.frame
+                                                            toView:nil]];
+            }
+
             [fullScreenWindow enterFullScreen];
             fullScreenEnabled = YES;
 
@@ -1149,8 +1169,6 @@ - (void)enterFullScreen:(int)fuoptions backgroundColor:(NSColor *)back
             if (blurRadius != 0)
                 [MMWindow setBlurRadius:blurRadius onWindow:fullScreenWindow];
 
-            // The resize handle disappears so the vim view needs to update the
-            // scrollbars.
             shouldResizeVimView = YES;
         }
     }
@@ -1664,8 +1682,6 @@ - (BOOL)readSelectionFromPasteboard:(NSPasteboard *)pboard
 }
 
 
-#if (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7)
-
 // -- Full-screen delegate ---------------------------------------------------
 
 - (NSApplicationPresentationOptions)window:(NSWindow *)window
@@ -1795,8 +1811,6 @@ - (void)windowDidFailToExitFullScreen:(NSWindow *)window
     [vimController addVimInput:@"<C-\\><C-N>:set fu<CR>"];
 }
 
-#endif // (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7)
-
 - (void)runAfterWindowPresentedUsingBlock:(void (^)(void))block
 {
     if (windowPresented) { // no need to defer block, just run it now

src/MacVim/MacVimTests/MacVimTests.m

@@ -17,6 +17,7 @@
 #import "MMApplication.h"
 #import "MMFullScreenWindow.h"
 #import "MMWindow.h"
+#import "MMTabline.h"
 #import "MMTextView.h"
 #import "MMWindowController.h"
 #import "MMVimController.h"
@@ -887,6 +888,8 @@ - (void) testResizeVimView {
     XCTAssertLessThan(textView.pendingMaxRows, 30); // confirms that we have an outstanding resize request to make it smaller
     XCTAssertLessThan(textView.pendingMaxColumns, 80);
     XCTAssertTrue(win.isRenderBlocked);
+    XCTAssertEqual(textView.drawRectOffset.width, 0);
+    XCTAssertEqual(textView.drawRectOffset.height, 0);
     // Vim has responded to the size change. We should now have unblocked rendering.
     [self waitForVimMessage:SetTextDimensionsNoResizeWindowMsgID blockFutureMessages:YES];
     XCTAssertLessThan(textView.maxRows, 30);
@@ -910,7 +913,7 @@ - (void) testResizeVimView {
     [self waitForVimMessage:ShowTabBarMsgID blockFutureMessages:YES];
     XCTAssertEqual(textView.maxRows, 30);
     XCTAssertLessThan(textView.pendingMaxRows, 30);
-    XCTAssertGreaterThan(textView.drawRectOffset.height, 0);
+    XCTAssertEqual(textView.drawRectOffset.height, MMTablineHeight);
     XCTAssertTrue(win.isRenderBlocked);
     [self waitForVimMessage:SetTextDimensionsNoResizeWindowMsgID blockFutureMessages:YES];
     XCTAssertLessThan(textView.maxRows, 30);
@@ -923,7 +926,11 @@ - (void) testResizeVimView {
     // was not explicitly set.
     [self setDefault:MMNativeFullScreenKey toValue:@NO]; // non-native is faster so use that
     [self sendStringToVim:@":set guioptions-=k fullscreen\n" withMods:0];
+    [self waitForVimMessage:EnterFullScreenMsgID blockFutureMessages:YES];
+    XCTAssertTrue(win.isRenderBlocked);
+    [self blockVimProcessInput:NO];
     [self waitForFullscreenTransitionIsEnter:YES isNative:NO];
+    XCTAssertFalse(win.isRenderBlocked);
     int fuRows = textView.maxRows;
     int fuCols = textView.maxColumns;
     [self sendStringToVim:@":set guifont=Menlo:h13\n" withMods:0];