Fix display mode switching and use separate X connection for display mode switching on linux

src/java/org/lwjgl/opengl/Display.java

@@ -158,18 +158,19 @@ public final class Display {
 	public static void setDisplayMode(DisplayMode mode) throws LWJGLException {
 		if (mode == null)
 			throw new NullPointerException("mode must be non-null");
+		current_mode = mode;
 		if (isCreated()) {
 			destroyWindow();
-			if (fullscreen)
-				switchDisplayMode(current_mode);
 			try {
+				if (fullscreen)
+					switchDisplayMode(mode);
 				createWindow();
 			} catch (LWJGLException e) {
+				destroyContext();
 				reset();
 				throw e;
 			}
 		}
-		current_mode = mode;
 	}
 
 	/**
@@ -361,17 +362,11 @@ public final class Display {
 			if (!isCreated())
 				return;
 			destroyWindow();
-			if (fullscreen) {
-				try {
-					switchDisplayMode(current_mode);
-				} catch (LWJGLException e) {
-					destroyContext();
-					throw e;
-				}
-			} else {
-				reset();
-			}
 			try {
+				if (fullscreen)
+					switchDisplayMode(current_mode);
+				else
+					resetDisplayMode();
 				createWindow();
 			} catch (LWJGLException e) {
 				destroyContext();
@@ -656,13 +651,14 @@ public final class Display {
 	 * in the static constructor
 	 */
 	private static void reset() {
-		if (!current_mode.equals(initial_mode))
+		if (!current_mode.equals(initial_mode)) {
 			resetDisplayMode();
-		current_mode = initial_mode;
+			current_mode = initial_mode;
+		}
 	}
 	
 	/**
-	 * @return the unique DIsplay context (or null, if the Display has not been created)
+	 * @return the unique Display context (or null, if the Display has not been created)
 	 */
 	public static Object getContext() {
 		return context;

src/native/linux/display.cpp

@@ -102,6 +102,7 @@ static bool setMode(Display *disp, int screen, int width, int height, bool lock_
 		}
 	}
 	XFree(avail_modes);
+	XFlush(disp);
 	return false;
 }
 
@@ -132,9 +133,11 @@ jobject initDisplay(JNIEnv *env) {
         int num_modes;
         XF86VidModeModeInfo **avail_modes;
 	int screen;
-	Display *disp = incDisplay(env);
-        if (disp == NULL)
+	Display *disp = XOpenDisplay(NULL);
+	if (disp == NULL) {
+		throwException(env, "Could not open display");
 		return NULL;
+	}
 	screen = DefaultScreen(disp);
 
 	if (!getDisplayModes(disp, screen, &num_modes, &avail_modes)) {
@@ -159,7 +162,7 @@ jobject initDisplay(JNIEnv *env) {
 		if (!XF86VidModeGetGammaRamp(disp, screen, gamma_ramp_length, r_ramp, g_ramp, b_ramp))
 			freeSavedGammaRamps();
 	}
-	decDisplay();
+	XCloseDisplay(disp);
 	return newMode;
 }
 
@@ -174,45 +177,49 @@ void switchDisplayMode(JNIEnv * env, jobject mode) {
 	int width = env->GetIntField(mode, fid_width);
 	int height = env->GetIntField(mode, fid_height);
 	int screen;
-	Display *disp = incDisplay(env);
-
-	if (disp == NULL)
+	Display *disp = XOpenDisplay(NULL);
+	if (disp == NULL) {
+		throwException(env, "Could not open display");
 		return;
+	}
 	screen = DefaultScreen(disp);
 	if (!setMode(disp, screen, width, height, true))
 		throwException(env, "Could not switch mode.");
-	decDisplay();
+	XCloseDisplay(disp);
 }
 
 void resetDisplayMode(JNIEnv *env) {
 	int screen;
-	Display *disp = incDisplay(env);
+	Display *disp = XOpenDisplay(NULL);
 	if (disp == NULL)
 		return;
 	screen = DefaultScreen(disp);
-	setMode(disp, screen, saved_width, saved_height, false);
+	if (!setMode(disp, screen, saved_width, saved_height, false)) {
+		printfDebug("Failed to reset mode");
+	}
 	if (gamma_ramp_length > 0) {
 		XF86VidModeSetGammaRamp(disp, screen, gamma_ramp_length, r_ramp, g_ramp, b_ramp);
 		freeSavedGammaRamps();
 	}
-	decDisplay();
+	XCloseDisplay(disp);
 }
 
 jobjectArray getAvailableDisplayModes(JNIEnv * env) {
 	int num_modes, i;
 	int screen;
 	XF86VidModeModeInfo **avail_modes;
-	Display *disp = incDisplay(env);
-
-	if (disp == NULL)
+	Display *disp = XOpenDisplay(NULL);
+	if (disp == NULL) {
+		throwException(env, "Could not open display");
 		return NULL;
+	}
 
 	screen = DefaultScreen(disp);
 	int bpp = XDefaultDepth(disp, screen);
 
 	if (!getDisplayModes(disp, screen, &num_modes, &avail_modes)) {
 		printfDebug("Could not get display modes\n");
-		decDisplay();
+		XCloseDisplay(disp);
 		return NULL;
 	}
 	// Allocate an array of DisplayModes big enough
@@ -225,7 +232,7 @@ jobjectArray getAvailableDisplayModes(JNIEnv * env) {
 		env->SetObjectArrayElement(ret, i, displayMode);
 	}
 	XFree(avail_modes);
-	decDisplay();
+	XCloseDisplay(disp);
 	return ret;
 }
 
@@ -238,9 +245,11 @@ void setGammaRamp(JNIEnv *env, jobject gamma_ramp_buffer) {
 		throwException(env, "gamma ramp length == 0.");
 		return;
 	}
-	Display * disp = incDisplay(env);
-	if (disp == NULL)
+	Display * disp = XOpenDisplay(NULL);
+	if (disp == NULL) {
+		throwException(env, "Could not open display");
 		return;
+	}
 	int screen = DefaultScreen(disp);
 	const float *gamma_ramp = (const float *)env->GetDirectBufferAddress(gamma_ramp_buffer);
 	unsigned short *ramp;
@@ -252,5 +261,5 @@ void setGammaRamp(JNIEnv *env, jobject gamma_ramp_buffer) {
 	if (XF86VidModeSetGammaRamp(disp, screen, gamma_ramp_length, ramp, ramp, ramp) == False) {
 		throwException(env, "Could not set gamma ramp.");
 	}
-	decDisplay();
+	XCloseDisplay(disp);
 }

src/native/linux/org_lwjgl_opengl_Display.cpp

@@ -130,14 +130,17 @@ Display *incDisplay(JNIEnv *env) {
 		}
 		warp_atom = XInternAtom(getDisplay(), "ignore_warp_atom", False);
 	}
+	async_x_error = false;
 	display_connection_usage++;
 	return display_connection;
 }
 
 void decDisplay(void) {
 	display_connection_usage--;
-	if (display_connection_usage == 0)
+	if (display_connection_usage == 0) {
 		XCloseDisplay(display_connection);
+		display_connection = NULL;
+	}
 }
 
 static void waitMapped(Window win) {