diff --git a/drivers/gpu/drm/vc4/vc4_plane.c b/drivers/gpu/drm/vc4/vc4_plane.c
index 2575bd81a010..3922f5f5807d 100644
--- a/drivers/gpu/drm/vc4/vc4_plane.c
+++ b/drivers/gpu/drm/vc4/vc4_plane.c
@@ -861,18 +861,21 @@ static int vc4_prepare_fb(struct drm_plane *plane,
 	struct dma_fence *fence;
 	int ret;
 
-	if ((plane->state->fb == state->fb) || !state->fb)
+	if (!state->fb)
 		return 0;
 
 	bo = to_vc4_bo(&drm_fb_cma_get_gem_obj(state->fb, 0)->base);
 
+	fence = reservation_object_get_excl_rcu(bo->resv);
+	drm_atomic_set_fence_for_plane(state, fence);
+
+	if (plane->state->fb == state->fb)
+		return 0;
+
 	ret = vc4_bo_inc_usecnt(bo);
 	if (ret)
 		return ret;
 
-	fence = reservation_object_get_excl_rcu(bo->resv);
-	drm_atomic_set_fence_for_plane(state, fence);
-
 	return 0;
 }