diff --git a/daemon/libvirtd.c b/daemon/libvirtd.c
index 0b5ae35068..1e91a4545e 100644
--- a/daemon/libvirtd.c
+++ b/daemon/libvirtd.c
@@ -363,14 +363,39 @@ static void daemonInitialize(void)
      * If they try to open a connection for a module that
      * is not loaded they'll get a suitable error at that point
      */
+# ifdef WITH_NETWORK
     virDriverLoadModule("network");
+# endif
+# ifdef WITH_STORAGE
     virDriverLoadModule("storage");
+# endif
+# ifdef WITH_NODE_DEVICES
     virDriverLoadModule("nodedev");
+# endif
+# ifdef WITH_SECRETS
     virDriverLoadModule("secret");
-    virDriverLoadModule("qemu");
-    virDriverLoadModule("lxc");
-    virDriverLoadModule("uml");
+# endif
+# ifdef WITH_NWFILTER
     virDriverLoadModule("nwfilter");
+# endif
+# ifdef WITH_INTERFACE
+    virDriverLoadModule("interface");
+# endif
+# ifdef WITH_QEMU
+    virDriverLoadModule("qemu");
+# endif
+# ifdef WITH_LXC
+    virDriverLoadModule("lxc");
+# endif
+# ifdef WITH_UML
+    virDriverLoadModule("uml");
+# endif
+# ifdef WITH_XEN
+    virDriverLoadModule("xen");
+# endif
+# ifdef WITH_LIBXL
+    virDriverLoadModule("libxl");
+# endif
 #else
 # ifdef WITH_NETWORK
     networkRegister();
@@ -381,7 +406,7 @@ static void daemonInitialize(void)
 # ifdef WITH_STORAGE_DIR
     storageRegister();
 # endif
-# if defined(WITH_NODE_DEVICES)
+# ifdef WITH_NODE_DEVICES
     nodedevRegister();
 # endif
 # ifdef WITH_SECRETS
diff --git a/src/Makefile.am b/src/Makefile.am
index e9621c11af..ff81d4cc56 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -626,27 +626,16 @@ libvirt_driver_la_LIBADD = $(NUMACTL_LIBS) $(GNUTLS_LIBS) $(DLOPEN_LIBS)
 USED_SYM_FILES = libvirt_private.syms
 
 if WITH_TEST
-if WITH_DRIVER_MODULES
-mod_LTLIBRARIES += libvirt_driver_test.la
-else
 noinst_LTLIBRARIES += libvirt_driver_test.la
 libvirt_la_BUILT_LIBADD += libvirt_driver_test.la
-endif
 libvirt_driver_test_la_CFLAGS = \
 		-I$(top_srcdir)/src/conf $(AM_CFLAGS)
-if WITH_DRIVER_MODULES
-libvirt_driver_test_la_LDFLAGS = -module -avoid-version $(AM_LDFLAGS)
-endif
 libvirt_driver_test_la_SOURCES = $(TEST_DRIVER_SOURCES)
 endif
 
 if WITH_REMOTE
-if WITH_DRIVER_MODULES
-mod_LTLIBRARIES += libvirt_driver_remote.la
-else
 noinst_LTLIBRARIES += libvirt_driver_remote.la
 libvirt_la_BUILT_LIBADD += libvirt_driver_remote.la
-endif
 libvirt_driver_remote_la_CFLAGS =				\
 		$(GNUTLS_CFLAGS)				\
 		$(XDR_CFLAGS)					\
@@ -658,10 +647,6 @@ libvirt_driver_remote_la_LIBADD = $(GNUTLS_LIBS) \
                 libvirt-net-rpc-client.la \
                 libvirt-net-rpc-server.la \
                 libvirt-net-rpc.la
-if WITH_DRIVER_MODULES
-libvirt_driver_remote_la_LIBADD += ../gnulib/lib/libgnu.la
-libvirt_driver_remote_la_LDFLAGS += -module -avoid-version
-endif
 libvirt_driver_remote_la_SOURCES = $(REMOTE_DRIVER_SOURCES)
 
 $(srcdir)/remote/remote_driver.c: $(REMOTE_DRIVER_GENERATED)
@@ -681,7 +666,8 @@ if WITH_DRIVER_MODULES
 mod_LTLIBRARIES += libvirt_driver_xen.la
 else
 noinst_LTLIBRARIES += libvirt_driver_xen.la
-libvirt_la_BUILT_LIBADD += libvirt_driver_xen.la
+# Stateful, so linked to daemon instead
+#libvirt_la_BUILT_LIBADD += libvirt_driver_xen.la
 endif
 libvirt_driver_xen_la_CFLAGS =					\
 		$(XEN_CFLAGS)					\
@@ -698,86 +684,46 @@ libvirt_driver_xen_la_SOURCES = $(XEN_DRIVER_SOURCES)
 endif
 
 if WITH_PHYP
-if WITH_DRIVER_MODULES
-mod_LTLIBRARIES += libvirt_driver_phyp.la
-else
 noinst_LTLIBRARIES += libvirt_driver_phyp.la
 libvirt_la_BUILT_LIBADD += libvirt_driver_phyp.la
-endif
 libvirt_driver_phyp_la_LIBADD = $(LIBSSH2_LIBS)
-if WITH_DRIVER_MODULES
-libvirt_driver_phyp_la_LIBADD += ../gnulib/lib/libgnu.la
-libvirt_driver_phyp_la_LDFLAGS = -module -avoid-version $(AM_LDFLAGS)
-endif
 libvirt_driver_phyp_la_CFLAGS = $(LIBSSH2_CFLAGS) \
 		-I$(top_srcdir)/src/conf $(AM_CFLAGS)
 libvirt_driver_phyp_la_SOURCES = $(PHYP_DRIVER_SOURCES)
 endif
 
 if WITH_OPENVZ
-if WITH_DRIVER_MODULES
-mod_LTLIBRARIES += libvirt_driver_openvz.la
-else
 noinst_LTLIBRARIES += libvirt_driver_openvz.la
 libvirt_la_BUILT_LIBADD += libvirt_driver_openvz.la
-endif
 libvirt_driver_openvz_la_CFLAGS = \
 		-I$(top_srcdir)/src/conf $(AM_CFLAGS)
-if WITH_DRIVER_MODULES
-libvirt_driver_openvz_la_LIBADD = ../gnulib/lib/libgnu.la
-libvirt_driver_openvz_la_LDFLAGS = -module -avoid-version $(AM_LDFLAGS)
-endif
 libvirt_driver_openvz_la_SOURCES = $(OPENVZ_DRIVER_SOURCES)
 endif
 
 if WITH_VMWARE
-if WITH_DRIVER_MODULES
-mod_LTLIBRARIES += libvirt_driver_vmware.la
-else
 noinst_LTLIBRARIES += libvirt_driver_vmware.la
 libvirt_la_BUILT_LIBADD += libvirt_driver_vmware.la
-endif
 libvirt_driver_vmware_la_CFLAGS = \
 		-I$(top_srcdir)/src/conf -I$(top_srcdir)/src/vmx $(AM_CFLAGS)
-if WITH_DRIVER_MODULES
-libvirt_driver_vmware_la_LIBADD = ../gnulib/lib/libgnu.la
-libvirt_driver_vmware_la_LDFLAGS = -module -avoid-version
-endif
 libvirt_driver_vmware_la_SOURCES = $(VMWARE_DRIVER_SOURCES)
 endif
 
 if WITH_VBOX
-if WITH_DRIVER_MODULES
-mod_LTLIBRARIES += libvirt_driver_vbox.la
-else
 noinst_LTLIBRARIES += libvirt_driver_vbox.la
 libvirt_la_BUILT_LIBADD += libvirt_driver_vbox.la
-endif
 libvirt_driver_vbox_la_CFLAGS = \
 		-I$(top_srcdir)/src/conf $(AM_CFLAGS)
 libvirt_driver_vbox_la_LIBADD = $(DLOPEN_LIBS) $(MSCOM_LIBS)
-if WITH_DRIVER_MODULES
-libvirt_driver_vbox_la_LIBADD += ../gnulib/lib/libgnu.la
-libvirt_driver_vbox_la_LDFLAGS = -module -avoid-version $(AM_LDFLAGS)
-endif
 libvirt_driver_vbox_la_SOURCES = $(VBOX_DRIVER_SOURCES)
 endif
 
 if WITH_XENAPI
-if WITH_DRIVER_MODULES
-mod_LTLIBRARIES += libvirt_driver_xenapi.la
-else
 noinst_LTLIBRARIES += libvirt_driver_xenapi.la
 libvirt_la_BUILT_LIBADD += libvirt_driver_xenapi.la
-endif
 libvirt_driver_xenapi_la_CFLAGS = $(LIBXENSERVER_CFLAGS) $(LIBCURL_CFLAGS) \
 		-I$(top_srcdir)/src/conf $(AM_CFLAGS)
 libvirt_driver_xenapi_la_LDFLAGS = $(AM_LDFLAGS)
 libvirt_driver_xenapi_la_LIBADD = $(LIBXENSERVER_LIBS) $(LIBCURL_LIBS)
-if WITH_DRIVER_MODULES
-libvirt_driver_xenapi_la_LIBADD += ../gnulib/lib/libgnu.la
-libvirt_driver_xenapi_la_LDFLAGS += -module -avoid-version
-endif
 libvirt_driver_xenapi_la_SOURCES = $(XENAPI_DRIVER_SOURCES)
 endif
 
@@ -896,20 +842,12 @@ $(ESX_DRIVER_GENERATED): $(srcdir)/esx/esx_vi_generator.input \
 	$(AM_V_GEN)srcdir=$(srcdir) $(PYTHON) $(srcdir)/esx/esx_vi_generator.py
 
 if WITH_ESX
-if WITH_DRIVER_MODULES
-mod_LTLIBRARIES += libvirt_driver_esx.la
-else
 noinst_LTLIBRARIES += libvirt_driver_esx.la
 libvirt_la_BUILT_LIBADD += libvirt_driver_esx.la
-endif
 libvirt_driver_esx_la_CFLAGS = $(LIBCURL_CFLAGS) \
 		-I$(top_srcdir)/src/conf -I$(top_srcdir)/src/vmx $(AM_CFLAGS)
 libvirt_driver_esx_la_LDFLAGS = $(AM_LDFLAGS)
 libvirt_driver_esx_la_LIBADD = $(LIBCURL_LIBS)
-if WITH_DRIVER_MODULES
-libvirt_driver_esx_la_LIBADD += ../gnulib/lib/libgnu.la
-libvirt_driver_esx_la_LDFLAGS += -module -avoid-version
-endif
 libvirt_driver_esx_la_SOURCES = $(ESX_DRIVER_SOURCES)
 libvirt_driver_esx_la_DEPENDENCIES = $(ESX_DRIVER_GENERATED)
 endif
@@ -922,20 +860,12 @@ $(HYPERV_DRIVER_GENERATED): $(srcdir)/hyperv/hyperv_wmi_generator.input \
 	$(AM_V_GEN)srcdir=$(srcdir) $(PYTHON) $(srcdir)/hyperv/hyperv_wmi_generator.py
 
 if WITH_HYPERV
-if WITH_DRIVER_MODULES
-mod_LTLIBRARIES += libvirt_driver_hyperv.la
-else
 noinst_LTLIBRARIES += libvirt_driver_hyperv.la
 libvirt_la_BUILT_LIBADD += libvirt_driver_hyperv.la
-endif
 libvirt_driver_hyperv_la_CFLAGS = $(OPENWSMAN_CFLAGS) \
 		-I$(top_srcdir)/src/conf $(AM_CFLAGS)
 libvirt_driver_hyperv_la_LDFLAGS = $(AM_LDFLAGS)
 libvirt_driver_hyperv_la_LIBADD = $(OPENWSMAN_LIBS)
-if WITH_DRIVER_MODULES
-libvirt_driver_hyperv_la_LIBADD += ../gnulib/lib/libgnu.la
-libvirt_driver_hyperv_la_LDFLAGS += -module -avoid-version
-endif
 libvirt_driver_hyperv_la_SOURCES = $(HYPERV_DRIVER_SOURCES)
 endif
 
@@ -966,7 +896,8 @@ if WITH_DRIVER_MODULES
 mod_LTLIBRARIES += libvirt_driver_interface.la
 else
 noinst_LTLIBRARIES += libvirt_driver_interface.la
-libvirt_la_BUILT_LIBADD += libvirt_driver_interface.la
+# Stateful, so linked to daemon instead
+#libvirt_la_BUILT_LIBADD += libvirt_driver_interface.la
 endif
 libvirt_driver_interface_la_CFLAGS = $(NETCF_CFLAGS) \
 		-I$(top_srcdir)/src/conf $(AM_CFLAGS)
diff --git a/src/libvirt.c b/src/libvirt.c
index 22fc863a7c..8900011c1e 100644
--- a/src/libvirt.c
+++ b/src/libvirt.c
@@ -45,37 +45,32 @@
 #include "virrandom.h"
 #include "viruri.h"
 
-#ifndef WITH_DRIVER_MODULES
-# ifdef WITH_TEST
-#  include "test/test_driver.h"
-# endif
-# ifdef WITH_XEN
-#  include "xen/xen_driver.h"
-# endif
-# ifdef WITH_REMOTE
-#  include "remote/remote_driver.h"
-# endif
-# ifdef WITH_OPENVZ
-#  include "openvz/openvz_driver.h"
-# endif
-# ifdef WITH_VMWARE
-#  include "vmware/vmware_driver.h"
-# endif
-# ifdef WITH_PHYP
-#  include "phyp/phyp_driver.h"
-# endif
-# ifdef WITH_VBOX
-#  include "vbox/vbox_driver.h"
-# endif
-# ifdef WITH_ESX
-#  include "esx/esx_driver.h"
-# endif
-# ifdef WITH_HYPERV
-#  include "hyperv/hyperv_driver.h"
-# endif
-# ifdef WITH_XENAPI
-#  include "xenapi/xenapi_driver.h"
-# endif
+#ifdef WITH_TEST
+# include "test/test_driver.h"
+#endif
+#ifdef WITH_REMOTE
+# include "remote/remote_driver.h"
+#endif
+#ifdef WITH_OPENVZ
+# include "openvz/openvz_driver.h"
+#endif
+#ifdef WITH_VMWARE
+# include "vmware/vmware_driver.h"
+#endif
+#ifdef WITH_PHYP
+# include "phyp/phyp_driver.h"
+#endif
+#ifdef WITH_VBOX
+# include "vbox/vbox_driver.h"
+#endif
+#ifdef WITH_ESX
+# include "esx/esx_driver.h"
+#endif
+#ifdef WITH_HYPERV
+# include "hyperv/hyperv_driver.h"
+#endif
+#ifdef WITH_XENAPI
+# include "xenapi/xenapi_driver.h"
 #endif
 
 #define VIR_FROM_THIS VIR_FROM_NONE
@@ -424,70 +419,32 @@ virInitialize(void)
      * Note that the order is important: the first ones have a higher
      * priority when calling virConnectOpen.
      */
-#ifdef WITH_DRIVER_MODULES
-    /* We don't care if any of these fail, because the whole point
-     * is to allow users to only install modules they want to use.
-     * If they try to open a connection for a module that
-     * is not loaded they'll get a suitable error at that point
-     */
-# ifdef WITH_TEST
-    virDriverLoadModule("test");
-# endif
-# ifdef WITH_XEN
-    virDriverLoadModule("xen");
-# endif
-# ifdef WITH_OPENVZ
-    virDriverLoadModule("openvz");
-# endif
-# ifdef WITH_VMWARE
-    virDriverLoadModule("vmware");
-# endif
-# ifdef WITH_VBOX
-    virDriverLoadModule("vbox");
-# endif
-# ifdef WITH_ESX
-    virDriverLoadModule("esx");
-# endif
-# ifdef WITH_HYPERV
-    virDriverLoadModule("hyperv");
-# endif
-# ifdef WITH_XENAPI
-    virDriverLoadModule("xenapi");
-# endif
-# ifdef WITH_REMOTE
-    virDriverLoadModule("remote");
-# endif
-#else
-# ifdef WITH_TEST
+#ifdef WITH_TEST
     if (testRegister() == -1) return -1;
-# endif
-# ifdef WITH_XEN
-    if (xenRegister () == -1) return -1;
-# endif
-# ifdef WITH_OPENVZ
+#endif
+#ifdef WITH_OPENVZ
     if (openvzRegister() == -1) return -1;
-# endif
-# ifdef WITH_VMWARE
+#endif
+#ifdef WITH_VMWARE
     if (vmwareRegister() == -1) return -1;
-# endif
-# ifdef WITH_PHYP
+#endif
+#ifdef WITH_PHYP
     if (phypRegister() == -1) return -1;
-# endif
-# ifdef WITH_VBOX
+#endif
+#ifdef WITH_VBOX
     if (vboxRegister() == -1) return -1;
-# endif
-# ifdef WITH_ESX
+#endif
+#ifdef WITH_ESX
     if (esxRegister() == -1) return -1;
-# endif
-# ifdef WITH_HYPERV
+#endif
+#ifdef WITH_HYPERV
     if (hypervRegister() == -1) return -1;
-# endif
-# ifdef WITH_XENAPI
+#endif
+#ifdef WITH_XENAPI
     if (xenapiRegister() == -1) return -1;
-# endif
-# ifdef WITH_REMOTE
+#endif
+#ifdef WITH_REMOTE
     if (remoteRegister () == -1) return -1;
-# endif
 #endif
 
     return 0;