diff --git a/worker/src/RTC/SimulcastConsumer.cpp b/worker/src/RTC/SimulcastConsumer.cpp
index 59b63ebcf9..d7d1fdfaa5 100644
--- a/worker/src/RTC/SimulcastConsumer.cpp
+++ b/worker/src/RTC/SimulcastConsumer.cpp
@@ -685,12 +685,13 @@ namespace RTC
 
 		// Check whether this is the packet we are waiting for in order to update
 		// the current spatial layer.
-		if (this->currentSpatialLayer != this->targetSpatialLayer && spatialLayer == this->targetSpatialLayer)
+		if (
+			// clang-format off
+			this->currentSpatialLayer != this->targetSpatialLayer &&
+			spatialLayer == this->targetSpatialLayer &&
+			packet->IsKeyFrame())
+			// clang-format on
 		{
-			// Ignore if not a key frame.
-			if (!packet->IsKeyFrame())
-				return;
-
 			shouldSwitchCurrentSpatialLayer = true;
 
 			// Need to resync the stream.