test: add a dlclose test

Closing an X display that had a VDPAU device created on it causes a crash.

Work around an identical libXext dlclose bug with the "Generic Event Extension"
by dlopening libXext.so.6 and leaving it open.

Original bug discovered and fixed by Robert Morell <rmorell@nvidia.com> in
commit 3b43955c

.

v2: Don't SKIP if creating the device fails.  Just attempting to create the
device installs the DRI2 extension that causes the problem.
Signed-off-by: Aaron Plattner <aplattner@nvidia.com>
Reviewed-by: Robert Morell <rmorell@nvidia.com>

.gitignore

@@ -24,4 +24,5 @@ missing
 *.la
 *.o
 stamp-h1
+test-driver
 vdpau.pc

Makefile.am

-SUBDIRS = doc src trace
+SUBDIRS = doc src test trace
 
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = vdpau.pc

configure.ac

@@ -81,5 +81,6 @@ XORG_CHANGELOG
 AC_OUTPUT([Makefile
            doc/Makefile
            src/Makefile
+           test/Makefile
            trace/Makefile
            vdpau.pc])

test/.gitignore

+dlclose

test/Makefile.am

+AM_CPPFLAGS = -I$(top_srcdir)/include
+CFLAGS = $(X11_CFLAGS)
+dlclose_LDADD = $(DLOPEN_LIBS) $(X11_LIBS)
+
+TESTS = dlclose
+check_PROGRAMS = $(TESTS)

test/dlclose.c

+#include <dlfcn.h>
+#include <stdio.h>
+#include <vdpau/vdpau.h>
+#include <vdpau/vdpau_x11.h>
+#include <X11/Xlib.h>
+
+#define PASS 0
+#define FAIL 1
+#define SKIP 77
+
+int main()
+{
+    // Work around a bug in libXext: dlclosing it after it has registered the
+    // Generic Event Extension causes an identical bug to the one this program
+    // is trying to test for.
+    void *libXext = dlopen("libXext.so.6", RTLD_LAZY);
+    void *libvdpau = dlopen("../src/.libs/libvdpau.so", RTLD_LAZY);
+    Display *dpy = XOpenDisplay(NULL);
+    VdpDeviceCreateX11 *pvdp_device_create_x11;
+    VdpDevice device;
+    VdpGetProcAddress *get_proc_address;
+    VdpStatus status;
+
+    if (!libXext) {
+        fprintf(stderr, "Failed to open libXext.so.6: %s", dlerror());
+        return SKIP;
+    }
+
+    if (!libvdpau) {
+        fprintf(stderr, "Failed to open libvdpau.so: %s", dlerror());
+        return FAIL;
+    }
+
+    if (!dpy) {
+        fprintf(stderr, "Failed to connect to X display %s\n", XDisplayName(NULL));
+        return SKIP;
+    }
+
+    pvdp_device_create_x11 = dlsym(libvdpau, "vdp_device_create_x11");
+    if (!pvdp_device_create_x11) {
+        fprintf(stderr, "Failed to find the symbol vdp_device_create_x11\n");
+        return FAIL;
+    }
+
+    status = pvdp_device_create_x11(dpy, 0, &device, &get_proc_address);
+    if (status == VDP_STATUS_OK) {
+        // It's okay if creating the device fails.  This will still install the
+        // DRI2 extension in libX11 and trigger the bug.
+        VdpDeviceDestroy *pvdp_device_destroy;
+
+        status = get_proc_address(device, VDP_FUNC_ID_DEVICE_DESTROY, (void**)&pvdp_device_destroy);
+        if (status != VDP_STATUS_OK) {
+            fprintf(stderr, "Failed to find the VdpDeviceDestroy function: %d\n", status);
+            return FAIL;
+        }
+
+        pvdp_device_destroy(device);
+    }
+
+    dlclose(libvdpau);
+    XCloseDisplay(dpy);
+
+    return PASS;
+}