diff --git a/quickstep/recents_ui_overrides/src/com/android/quickstep/BaseSwipeUpHandler.java b/quickstep/recents_ui_overrides/src/com/android/quickstep/BaseSwipeUpHandler.java
index b5fb31a9de..56af2e00c6 100644
--- a/quickstep/recents_ui_overrides/src/com/android/quickstep/BaseSwipeUpHandler.java
+++ b/quickstep/recents_ui_overrides/src/com/android/quickstep/BaseSwipeUpHandler.java
@@ -144,13 +144,14 @@ public abstract class BaseSwipeUpHandler<T extends StatefulActivity<?>, Q extend
                 TaskView nextTask = mRecentsView.getTaskView(taskId);
                 if (nextTask != null) {
                     mGestureState.updateLastStartedTaskId(taskId);
+                    boolean hasTaskPreviouslyAppeared = mGestureState.getPreviouslyAppearedTaskIds()
+                            .contains(taskId);
                     nextTask.launchTask(false /* animate */, true /* freezeTaskList */,
                             success -> {
                                 resultCallback.accept(success);
                                 if (success) {
-                                    if (mRecentsView.indexOfChild(nextTask)
-                                            == getLastAppearedTaskIndex()) {
-                                        onRestartLastAppearedTask();
+                                    if (hasTaskPreviouslyAppeared) {
+                                        onRestartPreviouslyAppearedTask();
                                     }
                                 } else {
                                     mActivityInterface.onLaunchTaskFailed();
@@ -171,7 +172,7 @@ public abstract class BaseSwipeUpHandler<T extends StatefulActivity<?>, Q extend
      * start A again to ensure it stays on top.
      */
     @CallSuper
-    protected void onRestartLastAppearedTask() {
+    protected void onRestartPreviouslyAppearedTask() {
         // Finish the controller here, since we won't get onTaskAppeared() for a task that already
         // appeared.
         if (mRecentsAnimationController != null) {
diff --git a/quickstep/recents_ui_overrides/src/com/android/quickstep/BaseSwipeUpHandlerV2.java b/quickstep/recents_ui_overrides/src/com/android/quickstep/BaseSwipeUpHandlerV2.java
index 414d7ae3fd..d55dc0f1b1 100644
--- a/quickstep/recents_ui_overrides/src/com/android/quickstep/BaseSwipeUpHandlerV2.java
+++ b/quickstep/recents_ui_overrides/src/com/android/quickstep/BaseSwipeUpHandlerV2.java
@@ -1122,8 +1122,8 @@ public abstract class BaseSwipeUpHandlerV2<T extends StatefulActivity<?>, Q exte
     }
 
     @Override
-    protected void onRestartLastAppearedTask() {
-        super.onRestartLastAppearedTask();
+    protected void onRestartPreviouslyAppearedTask() {
+        super.onRestartPreviouslyAppearedTask();
         reset();
     }
 
diff --git a/quickstep/recents_ui_overrides/src/com/android/quickstep/TouchInteractionService.java b/quickstep/recents_ui_overrides/src/com/android/quickstep/TouchInteractionService.java
index 4ca4e4c942..37314eaa3d 100644
--- a/quickstep/recents_ui_overrides/src/com/android/quickstep/TouchInteractionService.java
+++ b/quickstep/recents_ui_overrides/src/com/android/quickstep/TouchInteractionService.java
@@ -539,6 +539,8 @@ public class TouchInteractionService extends Service implements PluginListener<O
         if (mTaskAnimationManager.isRecentsAnimationRunning()) {
             gestureState.updateRunningTask(mGestureState.getRunningTask());
             gestureState.updateLastStartedTaskId(mGestureState.getLastStartedTaskId());
+            gestureState.updatePreviouslyAppearedTaskIds(
+                    mGestureState.getPreviouslyAppearedTaskIds());
         } else {
             gestureState.updateRunningTask(TraceHelper.whitelistIpcs("getRunningTask.0",
                     () -> mAM.getRunningTask(false /* filterOnlyVisibleRecents */)));
diff --git a/quickstep/src/com/android/quickstep/GestureState.java b/quickstep/src/com/android/quickstep/GestureState.java
index 188072a042..00b5eb9880 100644
--- a/quickstep/src/com/android/quickstep/GestureState.java
+++ b/quickstep/src/com/android/quickstep/GestureState.java
@@ -30,6 +30,8 @@ import com.android.systemui.shared.system.RemoteAnimationTargetCompat;
 
 import java.io.PrintWriter;
 import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Set;
 
 /**
  * Manages the state for an active system gesture, listens for events from the system and Launcher,
@@ -128,6 +130,7 @@ public class GestureState implements RecentsAnimationCallbacks.RecentsAnimationL
     private ActivityManager.RunningTaskInfo mRunningTask;
     private GestureEndTarget mEndTarget;
     private RemoteAnimationTargetCompat mLastAppearedTaskTarget;
+    private Set<Integer> mPreviouslyAppearedTaskIds = new HashSet<>();
     private int mLastStartedTaskId = -1;
 
     public GestureState(OverviewComponentObserver componentObserver, int gestureId) {
@@ -147,6 +150,7 @@ public class GestureState implements RecentsAnimationCallbacks.RecentsAnimationL
         mRunningTask = other.mRunningTask;
         mEndTarget = other.mEndTarget;
         mLastAppearedTaskTarget = other.mLastAppearedTaskTarget;
+        mPreviouslyAppearedTaskIds = other.mPreviouslyAppearedTaskIds;
         mLastStartedTaskId = other.mLastStartedTaskId;
     }
 
@@ -234,6 +238,9 @@ public class GestureState implements RecentsAnimationCallbacks.RecentsAnimationL
      */
     public void updateLastAppearedTaskTarget(RemoteAnimationTargetCompat lastAppearedTaskTarget) {
         mLastAppearedTaskTarget = lastAppearedTaskTarget;
+        if (lastAppearedTaskTarget != null) {
+            mPreviouslyAppearedTaskIds.add(lastAppearedTaskTarget.taskId);
+        }
     }
 
     /**
@@ -243,6 +250,14 @@ public class GestureState implements RecentsAnimationCallbacks.RecentsAnimationL
         return mLastAppearedTaskTarget != null ? mLastAppearedTaskTarget.taskId : -1;
     }
 
+    public void updatePreviouslyAppearedTaskIds(Set<Integer> previouslyAppearedTaskIds) {
+        mPreviouslyAppearedTaskIds = previouslyAppearedTaskIds;
+    }
+
+    public Set<Integer> getPreviouslyAppearedTaskIds() {
+        return mPreviouslyAppearedTaskIds;
+    }
+
     /**
      * Updates the last task that we started via startActivityFromRecents() during this gesture.
      */