diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index b5546a15206c..9be36ffb5689 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -9048,9 +9048,10 @@ static int load_balance(int this_cpu, struct rq *this_rq,
 out_balanced:
 	/*
 	 * We reach balance although we may have faced some affinity
-	 * constraints. Clear the imbalance flag if it was set.
+	 * constraints. Clear the imbalance flag only if other tasks got
+	 * a chance to move and fix the imbalance.
 	 */
-	if (sd_parent) {
+	if (sd_parent && !(env.flags & LBF_ALL_PINNED)) {
 		int *group_imbalance = &sd_parent->groups->sgc->imbalance;
 
 		if (*group_imbalance)