diff --git a/drivers/char/ipmi/ipmi_si_intf.c b/drivers/char/ipmi/ipmi_si_intf.c
index 5141ccf0b958..2b9f434775d4 100644
--- a/drivers/char/ipmi/ipmi_si_intf.c
+++ b/drivers/char/ipmi/ipmi_si_intf.c
@@ -232,6 +232,9 @@ struct smi_info {
 	/* Default driver model device. */
 	struct platform_device *pdev;
 
+	/* Have we added the device group to the device? */
+	bool dev_group_added;
+
 	/* Counters and things for the proc filesystem. */
 	atomic_t stats[SI_NUM_STATS];
 
@@ -2007,8 +2010,8 @@ int ipmi_si_add_smi(struct si_sm_io *io)
 	if (initialized) {
 		rv = try_smi_init(new_smi);
 		if (rv) {
-			mutex_unlock(&smi_infos_lock);
 			cleanup_one_si(new_smi);
+			mutex_unlock(&smi_infos_lock);
 			return rv;
 		}
 	}
@@ -2167,6 +2170,7 @@ static int try_smi_init(struct smi_info *new_smi)
 			rv);
 		goto out_err_stop_timer;
 	}
+	new_smi->dev_group_added = true;
 
 	rv = ipmi_register_smi(&handlers,
 			       new_smi,
@@ -2220,7 +2224,10 @@ static int try_smi_init(struct smi_info *new_smi)
 	return 0;
 
 out_err_remove_attrs:
-	device_remove_group(new_smi->io.dev, &ipmi_si_dev_attr_group);
+	if (new_smi->dev_group_added) {
+		device_remove_group(new_smi->io.dev, &ipmi_si_dev_attr_group);
+		new_smi->dev_group_added = false;
+	}
 	dev_set_drvdata(new_smi->io.dev, NULL);
 
 out_err_stop_timer:
@@ -2268,6 +2275,7 @@ static int try_smi_init(struct smi_info *new_smi)
 		else
 			platform_device_put(new_smi->pdev);
 		new_smi->pdev = NULL;
+		new_smi->io.dev = NULL;
 	}
 
 	kfree(init_name);
@@ -2364,8 +2372,10 @@ static void cleanup_one_si(struct smi_info *to_clean)
 		}
 	}
 
-	device_remove_group(to_clean->io.dev, &ipmi_si_dev_attr_group);
-	dev_set_drvdata(to_clean->io.dev, NULL);
+	if (to_clean->dev_group_added)
+		device_remove_group(to_clean->io.dev, &ipmi_si_dev_attr_group);
+	if (to_clean->io.dev)
+		dev_set_drvdata(to_clean->io.dev, NULL);
 
 	list_del(&to_clean->link);