Bug 578171 - [macOS 12] JVM crash when showing Shell during menu bar browsing

Change-Id: I2c2c1fc7566ea4b8e3b766601ec9badfa8381140
Signed-off-by: Alexandr Miloslavskiy <alexandr.miloslavskiy@syntevo.com>
Reviewed-on: https://git.eclipse.org/r/c/platform/eclipse.platform.swt/+/191209
Tested-by: Platform Bot <platform-bot@eclipse.org>
Tested-by: Lakshmi P Shanmugam <lshanmug@in.ibm.com>
Reviewed-by: Lakshmi P Shanmugam <lshanmug@in.ibm.com>
diff --git a/bundles/org.eclipse.swt/Eclipse SWT PI/cocoa/org/eclipse/swt/internal/cocoa/OS.java b/bundles/org.eclipse.swt/Eclipse SWT PI/cocoa/org/eclipse/swt/internal/cocoa/OS.java
index 3ec3241..e507f3c 100644
--- a/bundles/org.eclipse.swt/Eclipse SWT PI/cocoa/org/eclipse/swt/internal/cocoa/OS.java
+++ b/bundles/org.eclipse.swt/Eclipse SWT PI/cocoa/org/eclipse/swt/internal/cocoa/OS.java
@@ -21,6 +21,17 @@
 		Library.loadLibrary("swt-pi"); //$NON-NLS-1$
 	}
 
+	/*
+	 * macOS version, encoded as ((major << 16) + (minor << 8) + bugfix).
+	 *
+	 * Note that for macOS >= 11, it has "wrong" values if executable's
+	 * SDK is below 10.15 (or is it 11.0?), where 10.16.0 is reported
+	 * regardless of actual macOS version. On the other hand, with good
+	 * executable's SDK, real version is reported.
+	 *
+	 * This also means that executables with old SDK can't distinguish
+	 * macOS 11 from macOS 12.
+	 */
 	public static final int VERSION;
 
 	public static int VERSION (int major, int minor, int bugfix) {
@@ -190,10 +201,7 @@
 	 * @return true for macOS BigSur or later, returns false for macOS 10.15 and older
 	 */
 	public static boolean isBigSurOrLater () {
-		/*
-		 * Currently Big Sur OS version matches with 10.16 and not 11.0. This may be temporary.
-		 * Creating a method, so that it can be fixed in one place if/when this changes.
-		 */
+		// See comment for OS.VERSION for an explanation
 		return OS.VERSION >= OS.VERSION(10, 16, 0);
 	}
 
diff --git a/bundles/org.eclipse.swt/Eclipse SWT/cocoa/org/eclipse/swt/widgets/Display.java b/bundles/org.eclipse.swt/Eclipse SWT/cocoa/org/eclipse/swt/widgets/Display.java
index ccf7adb..d2ef223 100644
--- a/bundles/org.eclipse.swt/Eclipse SWT/cocoa/org/eclipse/swt/widgets/Display.java
+++ b/bundles/org.eclipse.swt/Eclipse SWT/cocoa/org/eclipse/swt/widgets/Display.java
@@ -860,8 +860,9 @@
 	 * macOS 11 always enables it regardless of sdk. The option is force
 	 * enabled here in case SWT runs with java/launcher linked with older sdk.
 	 */
-	if (!OS.isBigSurOrLater ())
+	if (!OS.isBigSurOrLater ()) {
 		configureSystemOption ("NSViewAllowsRootLayerBacking", true);
+	}
 
 	/*
 	 * Starting with macOS 11, layer backing is always enabled. That's fine.
@@ -874,8 +875,36 @@
 	 * things a lot slower. The workaround is to disable the "automatic" image
 	 * format.
 	 */
-	if (OS.isBigSurOrLater ())
+	if (OS.isBigSurOrLater ()) {
 		configureSystemOption ("NSViewUsesAutomaticLayerBackingStores", false);
+	}
+
+	/*
+	 * Bug 578171: There is new code in macOS 12 that remembers which
+	 * Shell was active before menu popup was shown and tries to
+	 * re-activate after menu popup is closed. Unfortunately there is a
+	 * bug in this code: if window list changes, it activates a wrong
+	 * Shell.
+	 *
+	 * This is a bug on its own, but worse yet, this causes a JVM crash
+	 * because activating a new Shell causes menu bar to reset its
+	 * internal data, which is unexpected to the macOS's menu tracking
+	 * loop.
+	 *
+	 * Both bugs are bugs of macOS itself. The workaround is to disable
+	 * the new macOS 12 behavior.
+	 *
+	 * The condition should be for (macOS >= 12), but it's not possible
+	 * to reliably distinguish 11 from 12, see comment for OS.VERSION.
+	 * That's fine: older macOS don't know this setting and will not
+	 * check for it anyway.
+	 */
+	if (OS.isBigSurOrLater ()) {
+		// The name of the option is misleading. What it really means
+		// is whether '-[NSMenuWindowManagerWindow _setVisible:]' shall
+		// save/restore current key window or not.
+		configureSystemOption ("NSMenuWindowManagerWindowShouldSetVisible", true);
+	}
 }
 
 /**
diff --git a/tests/org.eclipse.swt.tests/META-INF/MANIFEST.MF b/tests/org.eclipse.swt.tests/META-INF/MANIFEST.MF
index 4b7dae6..78d2b61 100644
--- a/tests/org.eclipse.swt.tests/META-INF/MANIFEST.MF
+++ b/tests/org.eclipse.swt.tests/META-INF/MANIFEST.MF
@@ -2,7 +2,7 @@
 Bundle-ManifestVersion: 2
 Bundle-Name: %pluginName
 Bundle-SymbolicName: org.eclipse.swt.tests
-Bundle-Version: 3.106.1600.qualifier
+Bundle-Version: 3.106.1700.qualifier
 Bundle-Vendor: %providerName
 Bundle-Localization: plugin
 Export-Package: org.eclipse.swt.tests.junit,
diff --git a/tests/org.eclipse.swt.tests/ManualTests/org/eclipse/swt/tests/manual/Bug578171_macOS_JvmCrash_ShowMenuWindow.java b/tests/org.eclipse.swt.tests/ManualTests/org/eclipse/swt/tests/manual/Bug578171_macOS_JvmCrash_ShowMenuWindow.java
new file mode 100644
index 0000000..2fd8c97
--- /dev/null
+++ b/tests/org.eclipse.swt.tests/ManualTests/org/eclipse/swt/tests/manual/Bug578171_macOS_JvmCrash_ShowMenuWindow.java
@@ -0,0 +1,78 @@
+/*******************************************************************************
+ * Copyright (c) 2022 Syntevo and others.
+ *
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *     Syntevo - initial API and implementation
+ *******************************************************************************/
+
+package org.eclipse.swt.tests.manual;
+
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.layout.*;
+import org.eclipse.swt.widgets.*;
+
+public class Bug578171_macOS_JvmCrash_ShowMenuWindow {
+	public static void main(String[] args) {
+		final Display display = new Display();
+		final Shell shell = new Shell(display);
+		shell.setLayout(new GridLayout(1, true));
+
+		new Label(shell, 0).setText(
+			"1) Use macOS 12 (macOS < 12 are not affected)\n" +
+			"2) Press the button\n" +
+			"3) Click menu bar and move mouse left-right to switch between submenus\n" +
+			"4) Bug 578171: Popup Shell will be activated; that's a bug already\n" +
+			"5) Bug 578171: JVM will crash"
+		);
+
+		Menu rootMenu = new Menu(shell, SWT.BAR);
+		shell.setMenuBar(rootMenu);
+		for (int iMenu = 0; iMenu < 3; iMenu++) {
+			MenuItem rootItem = new MenuItem(rootMenu, SWT.CASCADE);
+			rootItem.setText("Menu:" + iMenu);
+
+			Menu menu = new Menu(shell, SWT.DROP_DOWN);
+			rootItem.setMenu(menu);
+
+			for (int iItem = 0; iItem < 10; iItem++) {
+				MenuItem item = new MenuItem(menu, SWT.CASCADE);
+				item.setText("MenuItem:" + iMenu + ":" + iItem);
+			}
+		}
+
+		Button button = new Button(shell, SWT.PUSH);
+		button.setText("Show popup after 2000ms");
+		button.addListener(SWT.Selection, e -> {
+			display.timerExec(2000, () -> {
+				Shell popup = new Shell(shell, SWT.DIALOG_TRIM);
+				popup.addListener(SWT.Activate, e2 -> {
+					System.out.println("Popup Shell SWT.Activate: It's a bug when it happens without clicking the popup");
+				});
+
+				popup.setSize(100, 100);
+				popup.setVisible(true);
+			});
+		});
+
+		new Label(shell, 0).setText("A text field so that you can check if Shell is still active:");
+		new Text(shell, 0).setLayoutData(new GridData(SWT.FILL, SWT.FILL, true, true));
+
+		shell.pack();
+		shell.open();
+
+		while (!shell.isDisposed()) {
+			if (!display.readAndDispatch()) {
+				display.sleep();
+			}
+		}
+
+		display.dispose();
+	}
+}
\ No newline at end of file
diff --git a/tests/org.eclipse.swt.tests/pom.xml b/tests/org.eclipse.swt.tests/pom.xml
index f639377..997f934 100644
--- a/tests/org.eclipse.swt.tests/pom.xml
+++ b/tests/org.eclipse.swt.tests/pom.xml
@@ -19,7 +19,7 @@
   </parent>
   <groupId>org.eclipse.swt</groupId>
   <artifactId>org.eclipse.swt.tests</artifactId>
-  <version>3.106.1600-SNAPSHOT</version>
+  <version>3.106.1700-SNAPSHOT</version>
   <packaging>eclipse-test-plugin</packaging>
   <properties>
     <code.ignoredWarnings>${tests.ignoredWarnings}</code.ignoredWarnings>