diff --git a/src/libxl/libxl_driver.c b/src/libxl/libxl_driver.c
index 9955cc7bbb..bf97c9c116 100644
--- a/src/libxl/libxl_driver.c
+++ b/src/libxl/libxl_driver.c
@@ -3449,12 +3449,14 @@ libxlDomainDetachNetDevice(libxlDriverPrivatePtr driver,
         goto cleanup;
     }
 
-    networkReleaseActualDevice(vm->def, detach);
-    virDomainNetRemove(vm->def, detachidx);
     ret = 0;
 
  cleanup:
     libxl_device_nic_dispose(&nic);
+    if (!ret) {
+        networkReleaseActualDevice(vm->def, detach);
+        virDomainNetRemove(vm->def, detachidx);
+    }
     virObjectUnref(cfg);
     return ret;
 }