diff --git a/drivers/net/usb/hso.c b/drivers/net/usb/hso.c
index 5d885888658e..cb33fb4e9a80 100644
--- a/drivers/net/usb/hso.c
+++ b/drivers/net/usb/hso.c
@@ -2597,7 +2597,6 @@ static void hso_free_serial_device(struct hso_device *hso_dev)
 
 	if (!serial)
 		return;
-	set_serial_by_index(serial->minor, NULL);
 
 	hso_serial_common_free(serial);
 
@@ -3109,6 +3108,7 @@ static void hso_free_interface(struct usb_interface *interface)
 			mutex_unlock(&serial->parent->mutex);
 			hso_serial_tty_unregister(serial);
 			kref_put(&serial_table[i]->ref, hso_serial_ref_free);
+			set_serial_by_index(i, NULL);
 		}
 	}