diff --git a/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerFragment.kt b/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerFragment.kt
index bdf760204..f3f1cdeb8 100644
--- a/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerFragment.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerFragment.kt
@@ -47,11 +47,13 @@ class StoryViewerFragment : Fragment(R.layout.stories_viewer_fragment), StoryVie
 
   override fun onResume() {
     super.onResume()
+    viewModel.setIsScrolling(false)
     storyPager.registerOnPageChangeCallback(onPageChanged)
   }
 
   override fun onPause() {
     super.onPause()
+    viewModel.setIsScrolling(false)
     storyPager.unregisterOnPageChangeCallback(onPageChanged)
   }
 
@@ -67,6 +69,10 @@ class StoryViewerFragment : Fragment(R.layout.stories_viewer_fragment), StoryVie
     override fun onPageSelected(position: Int) {
       viewModel.setSelectedPage(position)
     }
+
+    override fun onPageScrollStateChanged(state: Int) {
+      viewModel.setIsScrolling(state == ViewPager2.SCROLL_STATE_DRAGGING)
+    }
   }
 
   companion object {
diff --git a/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerViewModel.kt
index 339d44ef6..f1aa6406a 100644
--- a/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerViewModel.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerViewModel.kt
@@ -1,6 +1,7 @@
 package org.thoughtcrime.securesms.stories.viewer
 
 import androidx.lifecycle.LiveData
+import androidx.lifecycle.MutableLiveData
 import androidx.lifecycle.ViewModel
 import androidx.lifecycle.ViewModelProvider
 import io.reactivex.rxjava3.disposables.CompositeDisposable
@@ -18,10 +19,17 @@ class StoryViewerViewModel(
 
   val state: LiveData<StoryViewerState> = store.stateLiveData
 
+  private val scrollStatePublisher: MutableLiveData<Boolean> = MutableLiveData(false)
+  val isScrolling: LiveData<Boolean> = scrollStatePublisher
+
   init {
     refresh()
   }
 
+  fun setIsScrolling(isScrolling: Boolean) {
+    scrollStatePublisher.value = isScrolling
+  }
+
   private fun refresh() {
     disposables.clear()
     disposables += repository.getStories().subscribe { recipientIds ->
diff --git a/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/page/StoryViewerPageFragment.kt b/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/page/StoryViewerPageFragment.kt
index 318039d58..c4744bdb2 100644
--- a/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/page/StoryViewerPageFragment.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/page/StoryViewerPageFragment.kt
@@ -37,6 +37,7 @@ import org.thoughtcrime.securesms.mediapreview.VideoControlsDelegate
 import org.thoughtcrime.securesms.recipients.Recipient
 import org.thoughtcrime.securesms.recipients.RecipientId
 import org.thoughtcrime.securesms.stories.dialogs.StoryContextMenu
+import org.thoughtcrime.securesms.stories.viewer.StoryViewerViewModel
 import org.thoughtcrime.securesms.stories.viewer.reply.direct.StoryDirectReplyDialogFragment
 import org.thoughtcrime.securesms.stories.viewer.reply.group.StoryGroupReplyBottomSheetDialogFragment
 import org.thoughtcrime.securesms.stories.viewer.reply.tabs.StoryViewsAndRepliesDialogFragment
@@ -67,6 +68,10 @@ class StoryViewerPageFragment : Fragment(R.layout.stories_viewer_fragment_page),
     }
   )
 
+  private val sharedViewModel: StoryViewerViewModel by viewModels(
+    ownerProducer = { requireParentFragment() }
+  )
+
   private val videoControlsDelegate = VideoControlsDelegate()
 
   private val lifecycleDisposable = LifecycleDisposable()
@@ -162,6 +167,25 @@ class StoryViewerPageFragment : Fragment(R.layout.stories_viewer_fragment_page),
       }
     }
 
+    sharedViewModel.isScrolling.observe(viewLifecycleOwner) { isScrolling ->
+      viewModel.setIsUserScrollingParent(isScrolling)
+    }
+
+    sharedViewModel.state.observe(viewLifecycleOwner) { parentState ->
+      if (parentState.pages.size <= parentState.page) {
+        viewModel.setIsSelectedPage(false)
+      } else if (storyRecipientId == parentState.pages[parentState.page]) {
+        if (progressBar.segmentCount != 0) {
+          progressBar.reset()
+          progressBar.setPosition(viewModel.getRestartIndex())
+          videoControlsDelegate.restart()
+        }
+        viewModel.setIsSelectedPage(true)
+      } else {
+        viewModel.setIsSelectedPage(false)
+      }
+    }
+
     viewModel.state.observe(viewLifecycleOwner) { state ->
       if (state.posts.isNotEmpty() && state.selectedPostIndex < state.posts.size) {
         val post = state.posts[state.selectedPostIndex]
@@ -221,15 +245,7 @@ class StoryViewerPageFragment : Fragment(R.layout.stories_viewer_fragment_page),
 
   override fun onPause() {
     super.onPause()
-  }
-
-  override fun onResume() {
-    super.onResume()
-
-    if (progressBar.segmentCount != 0) {
-      progressBar.reset()
-      progressBar.setPosition(viewModel.getRestartIndex())
-    }
+    pauseProgress()
   }
 
   override fun onFinishForwardAction() = Unit
diff --git a/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/page/StoryViewerPageViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/page/StoryViewerPageViewModel.kt
index 2ce46f6e9..17131eefe 100644
--- a/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/page/StoryViewerPageViewModel.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/page/StoryViewerPageViewModel.kt
@@ -73,6 +73,10 @@ class StoryViewerPageViewModel(
     }
   }
 
+  fun kickPlaybackState() {
+    storyViewerPlaybackStore.update { it }
+  }
+
   override fun onCleared() {
     disposables.clear()
   }
@@ -107,6 +111,14 @@ class StoryViewerPageViewModel(
     storyViewerDialogSubject.onNext(Optional.of(StoryViewerDialog.GroupDirectReply(recipientId, storyId)))
   }
 
+  fun setIsUserScrollingParent(isUserScrollingParent: Boolean) {
+    storyViewerPlaybackStore.update { it.copy(isUserScrollingParent = isUserScrollingParent) }
+  }
+
+  fun setIsSelectedPage(isSelectedPage: Boolean) {
+    storyViewerPlaybackStore.update { it.copy(isSelectedPage = isSelectedPage) }
+  }
+
   fun setIsDisplayingContextMenu(isDisplayingContextMenu: Boolean) {
     storyViewerPlaybackStore.update { it.copy(isDisplayingContextMenu = isDisplayingContextMenu) }
   }
diff --git a/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/page/StoryViewerPlaybackState.kt b/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/page/StoryViewerPlaybackState.kt
index 9e2925cb9..c50d1c172 100644
--- a/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/page/StoryViewerPlaybackState.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/page/StoryViewerPlaybackState.kt
@@ -8,7 +8,9 @@ data class StoryViewerPlaybackState(
   val isDisplayingContextMenu: Boolean = false,
   val isDisplayingViewsAndRepliesDialog: Boolean = false,
   val isDisplayingDirectReplyDialog: Boolean = false,
-  val isDisplayingCaptionOverlay: Boolean = false
+  val isDisplayingCaptionOverlay: Boolean = false,
+  val isUserScrollingParent: Boolean = false,
+  val isSelectedPage: Boolean = false
 ) {
   val isPaused: Boolean = !areSegmentsInitialized ||
     isUserTouching ||
@@ -18,5 +20,7 @@ data class StoryViewerPlaybackState(
     isDisplayingContextMenu ||
     isDisplayingViewsAndRepliesDialog ||
     isDisplayingDirectReplyDialog ||
-    isDisplayingCaptionOverlay
+    isDisplayingCaptionOverlay ||
+    isUserScrollingParent ||
+    !isSelectedPage
 }