diff --git a/src/core/recv_buffer.c b/src/core/recv_buffer.c index 87ac43b33c..5c10439abb 100644 --- a/src/core/recv_buffer.c +++ b/src/core/recv_buffer.c @@ -47,6 +47,19 @@ #include "recv_buffer.c.clog.h" #endif +_IRQL_requires_max_(DISPATCH_LEVEL) +void +QuicRecvChunkInitialize( + _Inout_ QUIC_RECV_CHUNK* Chunk, + _In_ uint32_t AllocLength, + _Inout_updates_(AllocLength) uint8_t* Buffer + ) +{ + Chunk->AllocLength = AllocLength; + Chunk->Buffer = Buffer; + Chunk->ExternalReference = FALSE; +} + _IRQL_requires_max_(DISPATCH_LEVEL) QUIC_STATUS // TODO - Can only fail if PreallocatedChunk == NULL QuicRecvBufferInitialize( @@ -57,47 +70,51 @@ QuicRecvBufferInitialize( _In_opt_ QUIC_RECV_CHUNK* PreallocatedChunk ) { - QUIC_STATUS Status; - - CXPLAT_DBG_ASSERT(AllocBufferLength != 0 && (AllocBufferLength & (AllocBufferLength - 1)) == 0); // Power of 2 - CXPLAT_DBG_ASSERT(VirtualBufferLength != 0 && (VirtualBufferLength & (VirtualBufferLength - 1)) == 0); // Power of 2 + CXPLAT_DBG_ASSERT(AllocBufferLength != 0 || RecvMode == QUIC_RECV_BUF_MODE_EXTERNAL); + CXPLAT_DBG_ASSERT(VirtualBufferLength != 0 || RecvMode == QUIC_RECV_BUF_MODE_EXTERNAL); + CXPLAT_DBG_ASSERT((AllocBufferLength & (AllocBufferLength - 1)) == 0); // Power of 2 + CXPLAT_DBG_ASSERT((VirtualBufferLength & (VirtualBufferLength - 1)) == 0); // Power of 2 CXPLAT_DBG_ASSERT(AllocBufferLength <= VirtualBufferLength); - QUIC_RECV_CHUNK* Chunk = NULL; - if (PreallocatedChunk != NULL) { - RecvBuffer->PreallocatedChunk = PreallocatedChunk; - Chunk = PreallocatedChunk; - } else { - RecvBuffer->PreallocatedChunk = NULL; - Chunk = CXPLAT_ALLOC_NONPAGED(sizeof(QUIC_RECV_CHUNK) + AllocBufferLength, QUIC_POOL_RECVBUF); - if (Chunk == NULL) { - QuicTraceEvent( - AllocFailure, - "Allocation of '%s' failed. (%llu bytes)", - "recv_buffer", - sizeof(QUIC_RECV_CHUNK) + AllocBufferLength); - Status = QUIC_STATUS_OUT_OF_MEMORY; - goto Error; - } - } - - QuicRangeInitialize(QUIC_MAX_RANGE_ALLOC_SIZE, &RecvBuffer->WrittenRanges); - CxPlatListInitializeHead(&RecvBuffer->Chunks); - CxPlatListInsertHead(&RecvBuffer->Chunks, &Chunk->Link); - Chunk->AllocLength = AllocBufferLength; - Chunk->ExternalReference = FALSE; RecvBuffer->BaseOffset = 0; RecvBuffer->ReadStart = 0; RecvBuffer->ReadPendingLength = 0; RecvBuffer->ReadLength = 0; - RecvBuffer->Capacity = AllocBufferLength; - RecvBuffer->VirtualBufferLength = VirtualBufferLength; RecvBuffer->RecvMode = RecvMode; - Status = QUIC_STATUS_SUCCESS; + QuicRangeInitialize(QUIC_MAX_RANGE_ALLOC_SIZE, &RecvBuffer->WrittenRanges); + CxPlatListInitializeHead(&RecvBuffer->Chunks); -Error: + if (RecvMode != QUIC_RECV_BUF_MODE_EXTERNAL) { + // + // Setup an initial chunk. + // + QUIC_RECV_CHUNK* Chunk = NULL; + if (PreallocatedChunk != NULL) { + RecvBuffer->PreallocatedChunk = PreallocatedChunk; + Chunk = PreallocatedChunk; + } else { + RecvBuffer->PreallocatedChunk = NULL; + Chunk = CXPLAT_ALLOC_NONPAGED(sizeof(QUIC_RECV_CHUNK) + AllocBufferLength, QUIC_POOL_RECVBUF); + if (Chunk == NULL) { + QuicTraceEvent( + AllocFailure, + "Allocation of '%s' failed. (%llu bytes)", + "recv_buffer", + sizeof(QUIC_RECV_CHUNK) + AllocBufferLength); + return QUIC_STATUS_OUT_OF_MEMORY; + } + QuicRecvChunkInitialize(Chunk, AllocBufferLength, (uint8_t*)(Chunk + 1)); + } + CxPlatListInsertHead(&RecvBuffer->Chunks, &Chunk->Link); + RecvBuffer->Capacity = AllocBufferLength; + RecvBuffer->VirtualBufferLength = VirtualBufferLength; + } else { + RecvBuffer->PreallocatedChunk = NULL; + RecvBuffer->Capacity = 0; + RecvBuffer->VirtualBufferLength = 0; + } - return Status; + return QUIC_STATUS_SUCCESS; } _IRQL_requires_max_(DISPATCH_LEVEL) @@ -164,14 +181,66 @@ QuicRecvBufferHasUnreadData( } _IRQL_requires_max_(DISPATCH_LEVEL) -void -QuicRecvBufferIncreaseVirtualBufferLength( +BOOLEAN +QuicRecvBufferTryIncreaseVirtualBufferLength( _In_ QUIC_RECV_BUFFER* RecvBuffer, _In_ uint32_t NewLength ) { + if (RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_EXTERNAL) { + return FALSE; + } + CXPLAT_DBG_ASSERT(NewLength >= RecvBuffer->VirtualBufferLength); // Don't support decrease. RecvBuffer->VirtualBufferLength = NewLength; + return TRUE; +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +QUIC_STATUS +QuicRecvBufferProvideChunks( + _Inout_ QUIC_RECV_BUFFER* RecvBuffer, + _Inout_ CXPLAT_LIST_ENTRY* /* QUIC_RECV_CHUNKS */ Chunks + ) +{ + CXPLAT_DBG_ASSERT(RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_EXTERNAL); + CXPLAT_DBG_ASSERT(!CxPlatListIsEmpty(Chunks)); + + uint64_t NewBufferLength = RecvBuffer->VirtualBufferLength; + for (CXPLAT_LIST_ENTRY* Link = Chunks->Flink; + Link != Chunks; + Link = Link->Flink) { + QUIC_RECV_CHUNK* Chunk = CXPLAT_CONTAINING_RECORD(Link, QUIC_RECV_CHUNK, Link); + NewBufferLength += Chunk->AllocLength; + if (NewBufferLength < Chunk->AllocLength) { + // + // Overflow: we can't handle that much buffer space. + // + return QUIC_STATUS_INVALID_PARAMETER; + } + } + + if (NewBufferLength > MAXUINT32) { + // + // We can't handle that much buffer space. + // + return QUIC_STATUS_INVALID_PARAMETER; + } + + if (CxPlatListIsEmpty(&RecvBuffer->Chunks)) { + // + // If a new chunk becomes the first chunk, update the capacity. + // + CXPLAT_DBG_ASSERT(RecvBuffer->ReadStart == 0); + CXPLAT_DBG_ASSERT(RecvBuffer->ReadLength == 0); + QUIC_RECV_CHUNK* firstChunk = CXPLAT_CONTAINING_RECORD(Chunks->Flink, QUIC_RECV_CHUNK, Link); + RecvBuffer->Capacity = firstChunk->AllocLength; + } + + RecvBuffer->VirtualBufferLength = (uint32_t)NewBufferLength; + CxPlatListMoveItems(Chunks, &RecvBuffer->Chunks); + + return QUIC_STATUS_SUCCESS; } // @@ -186,6 +255,7 @@ QuicRecvBufferResize( _In_ uint32_t TargetBufferLength ) { + CXPLAT_DBG_ASSERTMSG(RecvBuffer->RecvMode != QUIC_RECV_BUF_MODE_EXTERNAL, "Should never resize in External mode"); CXPLAT_DBG_ASSERT( TargetBufferLength != 0 && (TargetBufferLength & (TargetBufferLength - 1)) == 0); // Power of 2 @@ -209,8 +279,7 @@ QuicRecvBufferResize( return FALSE; } - NewChunk->AllocLength = TargetBufferLength; - NewChunk->ExternalReference = FALSE; + QuicRecvChunkInitialize(NewChunk, TargetBufferLength, (uint8_t*)(NewChunk + 1)); CxPlatListInsertTail(&RecvBuffer->Chunks, &NewChunk->Link); if (!LastChunk->ExternalReference) { @@ -306,7 +375,8 @@ QuicRecvBufferGetTotalAllocLength( _In_ QUIC_RECV_BUFFER* RecvBuffer ) { - if (RecvBuffer->RecvMode != QUIC_RECV_BUF_MODE_MULTIPLE) { + if (RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_SINGLE || + RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_CIRCULAR) { // // In single and circular mode, the last chunk is the only chunk being // written to at any given time, and therefore the only chunk we care @@ -321,30 +391,32 @@ QuicRecvBufferGetTotalAllocLength( } // - // For multiple mode, several chunks may be used at any point in time, so we - // need to consider the space allocated for all of them. Additionally, the - // first one is special because it may be used as a circular buffer, and - // already be partially drained. + // For multiple mode and external mode, several chunks may be used at any + // point in time, so we need to consider the space allocated for all of them. + // Additionally, the first one is special because it may be already partially + // drained, making it only partially usable. // QUIC_RECV_CHUNK* Chunk = CXPLAT_CONTAINING_RECORD( RecvBuffer->Chunks.Flink, QUIC_RECV_CHUNK, Link); - if (Chunk->Link.Flink == &RecvBuffer->Chunks) { + if (RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_MULTIPLE && + Chunk->Link.Flink == &RecvBuffer->Chunks) { // - // Only one chunk means we don't have an artificial "end", and will just - // write to the whole allocated length. + // In Multiple mode, only one chunk means we don't have an artificial + // "end", and are using the full allocated length of the buffer in a + // circular fashion. // return Chunk->AllocLength; } // - // When we have additional chunks following this, then its possible part of - // the first chunk has already been drained, so we don't use the allocated - // length, but ReadLength instead when calculating total available space. + // Otherwise, it is possible part of the first chunk has already been + // drained, so we don't use the allocated length, but the Capacity instead + // when calculating total available space. // - uint32_t AllocLength = RecvBuffer->ReadLength; + uint32_t AllocLength = RecvBuffer->Capacity; while (Chunk->Link.Flink != &RecvBuffer->Chunks) { Chunk = CXPLAT_CONTAINING_RECORD( @@ -368,9 +440,9 @@ QuicRecvBufferCopyIntoChunks( ) { // - // Copy the data into the correct chunk(s). In multiple mode this may result - // in copies to multiple buffers. For single/circular it should always be - // just a single copy. + // Copy the data into the correct chunk(s). In multiple/external mode this + // may result in copies to multiple chunks. For single/circular it should + // always be just a single copy. // // @@ -385,7 +457,8 @@ QuicRecvBufferCopyIntoChunks( WriteBuffer += Diff; } - if (RecvBuffer->RecvMode != QUIC_RECV_BUF_MODE_MULTIPLE) { + if (RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_SINGLE || + RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_CIRCULAR) { // // In single/circular mode we always just write to the last chunk. // @@ -407,12 +480,14 @@ QuicRecvBufferCopyIntoChunks( } if (Chunk->Link.Flink == &RecvBuffer->Chunks) { + // TODO guhetier: Isn't this always true by definition, since Chunk is the last chunk? + // Was this meant to update the ReadLength only if it is the first chunk instead? RecvBuffer->ReadLength = (uint32_t)(QuicRangeGet(&RecvBuffer->WrittenRanges, 0)->Count - RecvBuffer->BaseOffset); } } else { // - // In multiple mode we may have to write to multiple (two max) chunks. + // In multiple/external mode we may have to write to multiple chunks. // We need to find the first chunk to start writing at and then // continue copying data into the chunks until we run out. // @@ -426,14 +501,20 @@ QuicRecvBufferCopyIntoChunks( BOOLEAN IsFirstChunk = TRUE; uint64_t RelativeOffset = WriteOffset - RecvBuffer->BaseOffset; uint32_t ChunkOffset = RecvBuffer->ReadStart; - if (Chunk->Link.Flink == &RecvBuffer->Chunks) { + if (RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_MULTIPLE && + Chunk->Link.Flink == &RecvBuffer->Chunks) { + // + // In multiple mode, when there is only one chunk, it is used as a circular buffer: + // the whole allocated length is usable. + // CXPLAT_DBG_ASSERT(WriteLength <= Chunk->AllocLength); // Should always fit if we only have one ChunkLength = Chunk->AllocLength; RecvBuffer->ReadLength = (uint32_t)(QuicRangeGet(&RecvBuffer->WrittenRanges, 0)->Count - RecvBuffer->BaseOffset); } else { // - // In multiple mode, the first chunk may not start at the beginning. + // The first buffer might be partially drained and only the remaining capacity + // can be used. // ChunkLength = RecvBuffer->Capacity; @@ -466,13 +547,14 @@ QuicRecvBufferCopyIntoChunks( BOOLEAN IsFirstLoop = TRUE; do { + // TODO guhetier: No reason to have that in the loop, can be done once outside of it uint32_t ChunkWriteOffset = (ChunkOffset + RelativeOffset) % Chunk->AllocLength; if (!IsFirstChunk) { // This RelativeOffset is already shrunk to represent the offset from beginning of the current chunk. ChunkWriteOffset = (uint32_t)RelativeOffset; } if (!IsFirstLoop) { - // We are continue writing from previous chunk. So, start from the beginning of the currnet chunk. + // We continue writing from the previous chunk. So, start from the beginning of the current chunk. ChunkWriteOffset = 0; } @@ -486,6 +568,7 @@ QuicRecvBufferCopyIntoChunks( ChunkWriteLength = RecvBuffer->Capacity - (uint32_t)RelativeOffset; } if (Chunk->AllocLength < ChunkWriteOffset + ChunkWriteLength) { + CXPLAT_DBG_ASSERT(RecvBuffer->RecvMode != QUIC_RECV_BUF_MODE_EXTERNAL); // External mode capacity will never allow a wrap around // Circular buffer wrap around case. CxPlatCopyMemory(Chunk->Buffer + ChunkWriteOffset, WriteBuffer, Chunk->AllocLength - ChunkWriteOffset); CxPlatCopyMemory(Chunk->Buffer, WriteBuffer + Chunk->AllocLength - ChunkWriteOffset, ChunkWriteLength - (Chunk->AllocLength - ChunkWriteOffset)); @@ -573,24 +656,28 @@ QuicRecvBufferWrite( // N.B. We do this before updating the written ranges below so we don't have // to support rolling back those changes on the possible allocation failure // here. + // This is skipped in external mode since the entire virtual length is + // always allocated. // - uint32_t AllocLength = QuicRecvBufferGetTotalAllocLength(RecvBuffer); - if (AbsoluteLength > RecvBuffer->BaseOffset + AllocLength) { - // - // If we don't currently have enough room then we will want to resize - // the last chunk to be big enough to hold everything. We do this by - // repeatedly doubling its size until it is large enough. - // - uint32_t NewBufferLength = - CXPLAT_CONTAINING_RECORD( - RecvBuffer->Chunks.Blink, - QUIC_RECV_CHUNK, - Link)->AllocLength << 1; - while (AbsoluteLength > RecvBuffer->BaseOffset + NewBufferLength + RecvBuffer->ReadPendingLength) { - NewBufferLength <<= 1; - } - if (!QuicRecvBufferResize(RecvBuffer, NewBufferLength)) { - return QUIC_STATUS_OUT_OF_MEMORY; + if (RecvBuffer->RecvMode != QUIC_RECV_BUF_MODE_EXTERNAL) { + uint32_t AllocLength = QuicRecvBufferGetTotalAllocLength(RecvBuffer); + if (AbsoluteLength > RecvBuffer->BaseOffset + AllocLength) { + // + // If we don't currently have enough room then we will want to resize + // the last chunk to be big enough to hold everything. We do this by + // repeatedly doubling its size until it is large enough. + // + uint32_t NewBufferLength = + CXPLAT_CONTAINING_RECORD( + RecvBuffer->Chunks.Blink, + QUIC_RECV_CHUNK, + Link)->AllocLength << 1; + while (AbsoluteLength > RecvBuffer->BaseOffset + NewBufferLength + RecvBuffer->ReadPendingLength) { + NewBufferLength <<= 1; + } + if (!QuicRecvBufferResize(RecvBuffer, NewBufferLength)) { + return QUIC_STATUS_OUT_OF_MEMORY; + } } } @@ -632,6 +719,8 @@ QuicRecvBufferWrite( return QUIC_STATUS_SUCCESS; } +// TODO guhetier: This could return the number of buffers needed / extra buffer needed? +// For now, only fill up to the number of provided buffer and follow up _IRQL_requires_max_(DISPATCH_LEVEL) void QuicRecvBufferRead( @@ -644,12 +733,20 @@ QuicRecvBufferRead( { CXPLAT_DBG_ASSERT(QuicRangeGetSafe(&RecvBuffer->WrittenRanges, 0) != NULL); // Only fail if you call read before write indicates read ready. CXPLAT_DBG_ASSERT(!CxPlatListIsEmpty(&RecvBuffer->Chunks)); // Should always have at least one chunk + // + // Only MULTIPLE mode allows concurrent reads + // CXPLAT_DBG_ASSERT( RecvBuffer->ReadPendingLength == 0 || RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_MULTIPLE); + // + // Only MULTIPLE and EXTERNAL modes can have multiple chunks at read time. + // Other modes would coalesce chunks during a write or drain. + // CXPLAT_DBG_ASSERT( - RecvBuffer->Chunks.Flink->Flink == &RecvBuffer->Chunks || // Should only have one buffer if not using multiple receive mode - RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_MULTIPLE); + RecvBuffer->Chunks.Flink->Flink == &RecvBuffer->Chunks || + RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_MULTIPLE || + RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_EXTERNAL ); // // Find the length of the data written in the front, after the BaseOffset. @@ -712,7 +809,7 @@ QuicRecvBufferRead( Buffers[0].Buffer = Chunk->Buffer + ReadStart; } - } else { + } else if (RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_MULTIPLE) { CXPLAT_DBG_ASSERT(RecvBuffer->ReadPendingLength < ContiguousLength); // Shouldn't call read if there is nothing new to read uint64_t UnreadLength = ContiguousLength - RecvBuffer->ReadPendingLength; CXPLAT_DBG_ASSERT(UnreadLength > 0); @@ -800,6 +897,45 @@ QuicRecvBufferRead( } CXPLAT_DBG_ASSERT(TotalBuffersLength <= RecvBuffer->ReadPendingLength); #endif + } else { // RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_EXTERNAL + + uint64_t remainingDataToRead = ContiguousLength; + const uint32_t ProvidedBufferCount = *BufferCount; + *BufferCount = 0; + + // + // Read from the first chunk. + // + QUIC_RECV_CHUNK* Chunk = + CXPLAT_CONTAINING_RECORD( + RecvBuffer->Chunks.Flink, + QUIC_RECV_CHUNK, + Link); + Chunk->ExternalReference = TRUE; + Buffers[*BufferCount].Buffer = Chunk->Buffer + RecvBuffer->ReadStart; + Buffers[*BufferCount].Length = RecvBuffer->ReadLength; + remainingDataToRead -= RecvBuffer->ReadLength; + (*BufferCount)++; + + // + // Continue reading from the next chunks until we run out of buffers or data. + // + while (*BufferCount < ProvidedBufferCount && remainingDataToRead > 0) { + Chunk = + CXPLAT_CONTAINING_RECORD( + Chunk->Link.Flink, + QUIC_RECV_CHUNK, + Link); + + Chunk->ExternalReference = TRUE; + uint32_t ChunkReadLength = (uint32_t)min(Chunk->AllocLength, remainingDataToRead); + Buffers[*BufferCount].Buffer = Chunk->Buffer; + Buffers[*BufferCount].Length = ChunkReadLength; + remainingDataToRead -= ChunkReadLength; + (*BufferCount)++; + } + *BufferOffset = RecvBuffer->BaseOffset; + RecvBuffer->ReadPendingLength = ContiguousLength - remainingDataToRead; } } @@ -821,7 +957,8 @@ QuicRecvBufferPartialDrain( CXPLAT_DBG_ASSERT(Chunk->ExternalReference); if (Chunk->Link.Flink != &RecvBuffer->Chunks && - RecvBuffer->RecvMode != QUIC_RECV_BUF_MODE_MULTIPLE) { + (RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_SINGLE || + RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_CIRCULAR)) { // // In single/circular mode, if there is another chunk, then that means // we no longer need this chunk at all because the other chunk contains @@ -857,16 +994,19 @@ QuicRecvBufferPartialDrain( Chunk->Buffer + DrainLength, (size_t)(Chunk->AllocLength - (uint32_t)DrainLength)); // TODO - Might be able to copy less than the full alloc length - } else { // Circular and multiple mode. + } else { // Circular, multiple and external mode. // // Increment the buffer start, making sure to account for circular // buffer wrap around. // RecvBuffer->ReadStart = (uint32_t)((RecvBuffer->ReadStart + DrainLength) % Chunk->AllocLength); - if (Chunk->Link.Flink != &RecvBuffer->Chunks) { + if (RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_EXTERNAL || + Chunk->Link.Flink != &RecvBuffer->Chunks) { // - // If there is another chunk, then the capacity of first chunk is shrunk. + // Shrink the capacity of the first chunk in external mode or + // if there is another chunk (in circular and multiple mode, + // when there is a single chunk, it is used as a circular buffer). // RecvBuffer->Capacity -= (uint32_t)DrainLength; } @@ -890,6 +1030,14 @@ QuicRecvBufferPartialDrain( CXPLAT_DBG_ASSERT(DrainLength <= RecvBuffer->ReadPendingLength); RecvBuffer->ReadPendingLength -= DrainLength; } + + if (RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_EXTERNAL) { + // + // In EXTERNAL mode, memory is never re-used: a drain consumes + // virtual buffer length. + // + RecvBuffer->VirtualBufferLength -= RecvBuffer->ReadLength; + } } // @@ -905,6 +1053,7 @@ QuicRecvBufferFullDrain( ) { CXPLAT_DBG_ASSERT(!CxPlatListIsEmpty(&RecvBuffer->Chunks)); + QUIC_RECV_CHUNK* Chunk = CXPLAT_CONTAINING_RECORD( RecvBuffer->Chunks.Flink, @@ -919,43 +1068,63 @@ QuicRecvBufferFullDrain( if (RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_MULTIPLE) { RecvBuffer->ReadPendingLength -= RecvBuffer->ReadLength; } + if (RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_EXTERNAL) { + // + // In EXTERNAL mode, memory is never re-used: a drain consumes + // virtual buffer length. + // + RecvBuffer->VirtualBufferLength -= RecvBuffer->ReadLength; + } RecvBuffer->ReadLength = (uint32_t)(QuicRangeGet(&RecvBuffer->WrittenRanges, 0)->Count - RecvBuffer->BaseOffset); if (Chunk->Link.Flink == &RecvBuffer->Chunks) { // - // No more chunks to drain, so we should also be out of buffer length - // to drain too. Return TRUE to indicate all data has been drained. + // We are completely draining the last chunk we have: ensure we are not + // requested to drain more. // CXPLAT_FRE_ASSERTMSG(DrainLength == 0, "App drained more than was available!"); CXPLAT_DBG_ASSERT(RecvBuffer->ReadLength == 0); + + if (RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_EXTERNAL) { + // + // In EXTERNAL mode, external chunks are never re-used: + // free the last chunk. + // + CxPlatListEntryRemove(&Chunk->Link); + if (Chunk != RecvBuffer->PreallocatedChunk) { + CXPLAT_FREE(Chunk, QUIC_POOL_RECVBUF); + } + RecvBuffer->Capacity = 0; + } + return 0; } // - // Cleanup the chunk that was just drained. - // + // We have more chunks and just drained this one completely: we are never + // going to re-use this one. Free it. + // CxPlatListEntryRemove(&Chunk->Link); if (Chunk != RecvBuffer->PreallocatedChunk) { CXPLAT_FREE(Chunk, QUIC_POOL_RECVBUF); } - if (RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_MULTIPLE) { - // - // The rest of the contiguous data might not fit in just the next chunk - // so we need to update the ReadLength of the first chunk to be no more - // than the next chunk's allocation length. - // Capacity is also updated to reflect the new first chunk's allocation length. - // - Chunk = - CXPLAT_CONTAINING_RECORD( - RecvBuffer->Chunks.Flink, - QUIC_RECV_CHUNK, - Link); - RecvBuffer->Capacity = Chunk->AllocLength; - if (Chunk->AllocLength < RecvBuffer->ReadLength) { - RecvBuffer->ReadLength = Chunk->AllocLength; - } + // + // The rest of the contiguous data might not fit in just the next chunk + // so we need to update the ReadLength of the first chunk to be no more + // than the next chunk's allocation length. + // Capacity is also updated to reflect the new first chunk's allocation length. + // + // Update the ReadLength and Capacity to match the new first chunk. + Chunk = + CXPLAT_CONTAINING_RECORD( + RecvBuffer->Chunks.Flink, + QUIC_RECV_CHUNK, + Link); + RecvBuffer->Capacity = Chunk->AllocLength; + if (Chunk->AllocLength < RecvBuffer->ReadLength) { + RecvBuffer->ReadLength = Chunk->AllocLength; } return DrainLength; @@ -976,19 +1145,51 @@ QuicRecvBufferDrain( CXPLAT_DBG_ASSERT(FirstRange); CXPLAT_DBG_ASSERT(FirstRange->Low == 0); do { - BOOLEAN PartialDrain = (uint64_t)RecvBuffer->ReadLength > DrainLength; - if (PartialDrain || - (QuicRangeSize(&RecvBuffer->WrittenRanges) > 1 && - RecvBuffer->BaseOffset + RecvBuffer->ReadLength == FirstRange->Count)) { + // + // Whether all the available data has been drained or more is readily available. + // + BOOLEAN MoreDataReadable = (uint64_t)RecvBuffer->ReadLength > DrainLength; + BOOLEAN GapInChunk = QuicRangeSize(&RecvBuffer->WrittenRanges) > 1 && + RecvBuffer->BaseOffset + RecvBuffer->ReadLength == FirstRange->Count; + + // + // In single/circular mode, a full drain must be done only all the data + // written in the buffer got read. + // A partial drain is done if not all the readily readable data was read + // or if the read is limited by a gap in the data. + // + BOOLEAN PartialDrain = MoreDataReadable || GapInChunk; + if (RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_MULTIPLE) { + // + // In addition to the above, in multiple mode, a chunk must be fully + // drained if its capacity is entirely consumed. + // + PartialDrain &= (uint64_t)RecvBuffer->Capacity > DrainLength; + } else if (RecvBuffer->RecvMode == QUIC_RECV_BUF_MODE_EXTERNAL) { // - // If there are 2 or more written ranges in the first chunk, it means that there may be - // more data later in the chunk that couldn't be read because there is a gap. - // Reuse the partial drain logic to preserve data after the gap. + // In external mode, the chunk must be full drained only if its capacity reaches 0. + // Otherwise, we either have more bytes to read, or more space to write. + // Contrary to other modes, we can reset ReadStart to the start of the buffer whenever + // we drained all written data. + // + PartialDrain = (uint64_t)RecvBuffer->Capacity > DrainLength; + } + + if (PartialDrain) { + // + // In single/circular mode, a full drain must be done only all the data + // written to the buffer got read. + // A partial drain is done if not all the readily readable data was read + // or if the read is limited by a gap in the data. // QuicRecvBufferPartialDrain(RecvBuffer, DrainLength); - return !PartialDrain; + return !MoreDataReadable; } + // + // The chunk doesn't contain anything useful anymore, it can be + // discarded or reused without constraints. + // DrainLength = QuicRecvBufferFullDrain(RecvBuffer, DrainLength); } while (DrainLength != 0); diff --git a/src/core/recv_buffer.h b/src/core/recv_buffer.h index 932afc8cc1..3647f3e7f1 100644 --- a/src/core/recv_buffer.h +++ b/src/core/recv_buffer.h @@ -12,19 +12,31 @@ extern "C" { typedef enum QUIC_RECV_BUF_MODE { QUIC_RECV_BUF_MODE_SINGLE, // Only one receive with a single contiguous buffer at a time. QUIC_RECV_BUF_MODE_CIRCULAR, // Only one receive that may indicate two contiguous buffers at a time. - QUIC_RECV_BUF_MODE_MULTIPLE // Multiple independent receives that may indicate up to two contiguous buffers at a time. + QUIC_RECV_BUF_MODE_MULTIPLE, // Multiple independent receives that may indicate up to two contiguous buffers at a time. + QUIC_RECV_BUF_MODE_EXTERNAL // Uses memory buffer provided by the app. Only one receive at a time, + // that may indicate up to the number of provided buffers. } QUIC_RECV_BUF_MODE; // // Represents a single contiguous range of bytes. // typedef struct QUIC_RECV_CHUNK { - CXPLAT_LIST_ENTRY Link; // Link in the list of chunks. - uint32_t AllocLength : 31; // Allocation size of Buffer - uint32_t ExternalReference : 1; // Indicates the buffer is being used externally. - uint8_t Buffer[0]; + CXPLAT_LIST_ENTRY Link; // Link in the list of chunks. + uint32_t AllocLength : 31; // Allocation size of Buffer + uint32_t ExternalReference : 1; // Indicates the buffer is being used externally. + _Field_size_(AllocLength) uint8_t *Buffer; // Pointer to the buffer itself. Doesn't need to be freed independently: + // - for internally allocated buffers, points in the same allocation. + // - for exteral buffers, the buffer isn't owned } QUIC_RECV_CHUNK; +_IRQL_requires_max_(DISPATCH_LEVEL) +void +QuicRecvChunkInitialize( + _Inout_ QUIC_RECV_CHUNK* Chunk, + _In_ uint32_t AllocLength, + _Inout_updates_(AllocLength) uint8_t* Buffer + ); + typedef struct QUIC_RECV_BUFFER { // @@ -38,7 +50,10 @@ typedef struct QUIC_RECV_BUFFER { QUIC_RECV_CHUNK* PreallocatedChunk; // - // The ranges that currently have bytes written to them. + // Ranges of stream offsets that have been written to the buffer, + // starting from 0 (not only what is currently in the buffer). + // The first sub-range includes [0, BaseOffset + ReadLength), + // and potentially more if ReadLength is constrained by the chunk size. // QUIC_RANGE WrittenRanges; @@ -53,18 +68,21 @@ typedef struct QUIC_RECV_BUFFER { uint64_t BaseOffset; // - // Start of the head in the circular of the first chunk. + // Position of the reading head in the first chunk. // uint32_t ReadStart; // - // The length of data available to read in the first chunk, starting at - // ReadStart. + // The length of data available to read in the first "active" chunk, + // starting at ReadStart. + // ("Active" means a chunk that can targetted by a read or write. + // In SINGLE and CIRCULAR modes, only the last chunk is "active".) // uint32_t ReadLength; // // Length of the buffer indicated to peers. + // Invariant: BaseOffset + VirtualBufferLength must never decrease. // uint32_t VirtualBufferLength; @@ -122,12 +140,24 @@ QuicRecvBufferHasUnreadData( // Changes the buffer's virtual buffer length. // _IRQL_requires_max_(DISPATCH_LEVEL) -void -QuicRecvBufferIncreaseVirtualBufferLength( +BOOLEAN +QuicRecvBufferTryIncreaseVirtualBufferLength( _In_ QUIC_RECV_BUFFER* RecvBuffer, _In_ uint32_t NewLength ); +// +// Provide externally allocated chunks to the buffer. +// At least one chunk must be provided. +// Only valid for QUIC_RECV_BUF_MODE_EXTERNAL mode. +// +_IRQL_requires_max_(DISPATCH_LEVEL) +QUIC_STATUS +QuicRecvBufferProvideChunks( + _Inout_ QUIC_RECV_BUFFER* RecvBuffer, + _Inout_ CXPLAT_LIST_ENTRY* /* QUIC_RECV_CHUNKS */ Chunks + ); + // // Buffers a (possibly out-of-order or duplicate) range of bytes. // diff --git a/src/core/unittest/RecvBufferTest.cpp b/src/core/unittest/RecvBufferTest.cpp index bca7767609..9c0f6e8732 100644 --- a/src/core/unittest/RecvBufferTest.cpp +++ b/src/core/unittest/RecvBufferTest.cpp @@ -14,17 +14,24 @@ #include "RecvBufferTest.cpp.clog.h" #endif -#define DEF_TEST_BUFFER_LENGTH 64 -#define LARGE_TEST_BUFFER_LENGTH 1024 +#include +#include + +#define DEF_TEST_BUFFER_LENGTH 64u +#define LARGE_TEST_BUFFER_LENGTH 1024u struct RecvBuffer { QUIC_RECV_BUFFER RecvBuf {0}; QUIC_RECV_CHUNK* PreallocChunk {nullptr}; + uint8_t* ExternalBuffer {nullptr}; ~RecvBuffer() { QuicRecvBufferUninitialize(&RecvBuf); if (PreallocChunk) { CXPLAT_FREE(PreallocChunk, QUIC_POOL_TEST); } + if (ExternalBuffer) { + CXPLAT_FREE(ExternalBuffer, QUIC_POOL_TEST); + } } QUIC_STATUS Initialize( _In_ QUIC_RECV_BUF_MODE RecvMode = QUIC_RECV_BUF_MODE_SINGLE, @@ -39,7 +46,38 @@ struct RecvBuffer { QUIC_POOL_TEST); } printf("Initializing: [mode=%u,vlen=%u,alen=%u]\n", RecvMode, VirtualBufferLength, AllocBufferLength); - auto Result = QuicRecvBufferInitialize(&RecvBuf, AllocBufferLength, VirtualBufferLength, RecvMode, PreallocChunk); + + auto Result = QUIC_STATUS_SUCCESS; + Result = QuicRecvBufferInitialize(&RecvBuf, AllocBufferLength, VirtualBufferLength, RecvMode, PreallocChunk); + if (Result != QUIC_STATUS_SUCCESS) { + return Result; + } + + if (RecvMode == QUIC_RECV_BUF_MODE_EXTERNAL && AllocBufferLength > 0) { + // + // If the receive mode is EXTERNAL, provide external buffers. + // Provide up to two chunks, so that: + // - the first chunk has `AllocBufferLength` bytes + // - the sum of the two is `VirtualBufferLength` bytes + // + CXPLAT_LIST_ENTRY ChunkList; + CxPlatListInitializeHead(&ChunkList); + ExternalBuffer = (uint8_t *)CXPLAT_ALLOC_NONPAGED(VirtualBufferLength, QUIC_POOL_TEST); + auto Chunk = (QUIC_RECV_CHUNK *)CXPLAT_ALLOC_NONPAGED(sizeof(QUIC_RECV_CHUNK), QUIC_POOL_RECVBUF); + QuicRecvChunkInitialize(Chunk, AllocBufferLength, ExternalBuffer); + CxPlatListInsertHead(&ChunkList, &Chunk->Link); + if (VirtualBufferLength > AllocBufferLength) { + auto Chunk2 = (QUIC_RECV_CHUNK *)CXPLAT_ALLOC_NONPAGED(sizeof(QUIC_RECV_CHUNK), QUIC_POOL_RECVBUF); + QuicRecvChunkInitialize(Chunk2, VirtualBufferLength - AllocBufferLength, ExternalBuffer + AllocBufferLength); + CxPlatListInsertTail(&ChunkList, &Chunk2->Link); + } + Result = QuicRecvBufferProvideChunks(&RecvBuf, &ChunkList); + } + else + { + Result = QuicRecvBufferInitialize(&RecvBuf, AllocBufferLength, VirtualBufferLength, RecvMode, PreallocChunk); + } + Dump(); return Result; } @@ -49,8 +87,13 @@ struct RecvBuffer { bool HasUnreadData() { return QuicRecvBufferHasUnreadData(&RecvBuf) != FALSE; } + QUIC_STATUS ProvideChunks(_Inout_ CXPLAT_LIST_ENTRY* Chunks) { + auto Result = QuicRecvBufferProvideChunks(&RecvBuf, Chunks); + Dump(); + return Result; + } void IncreaseVirtualBufferLength(uint32_t Length) { - QuicRecvBufferIncreaseVirtualBufferLength(&RecvBuf, Length); + QuicRecvBufferTryIncreaseVirtualBufferLength(&RecvBuf, Length); } QUIC_STATUS Write( _In_ uint64_t WriteOffset, @@ -100,21 +143,14 @@ struct RecvBuffer { ) { ASSERT_EQ(RecvBuf.ReadStart, ReadStart); ASSERT_EQ(RecvBuf.ReadLength, ReadLength); - int ChunkCount = 1; - QUIC_RECV_CHUNK* Chunk = - CXPLAT_CONTAINING_RECORD( - RecvBuf.Chunks.Flink, - QUIC_RECV_CHUNK, - Link); - ASSERT_EQ(Chunk->ExternalReference, ExternalReferences[0]); - while (Chunk->Link.Flink != &RecvBuf.Chunks) { + + int ChunkCount = 0; + for (CXPLAT_LIST_ENTRY* Entry = RecvBuf.Chunks.Flink; + Entry != &RecvBuf.Chunks; + Entry = Entry->Flink) { + auto* Chunk = CXPLAT_CONTAINING_RECORD(Entry, QUIC_RECV_CHUNK, Link); + ASSERT_EQ(Chunk->ExternalReference, ExternalReferences[ChunkCount]); ChunkCount++; - Chunk = - CXPLAT_CONTAINING_RECORD( - Chunk->Link.Flink, - QUIC_RECV_CHUNK, - Link); - ASSERT_EQ(Chunk->ExternalReference, ExternalReferences[ChunkCount - 1]); } ASSERT_EQ(ChunkCount, NumChunks); } @@ -148,9 +184,12 @@ struct RecvBuffer { _In_ BOOLEAN* ExternalReferences ) { uint64_t ReadOffset; - QUIC_BUFFER ReadBuffers[3]; - uint32_t ActualBufferCount = ARRAYSIZE(ReadBuffers); - Read(&ReadOffset, &ActualBufferCount, ReadBuffers); + // + // Always provide at least 3 buffers since some modes assume that many + // + uint32_t ActualBufferCount = std::max(BufferCount, 3u); + std::vector ReadBuffers(ActualBufferCount); + Read(&ReadOffset, &ActualBufferCount, ReadBuffers.data()); ASSERT_EQ(BufferCount, ActualBufferCount); for (uint32_t i = 0; i < ActualBufferCount; ++i) { @@ -515,8 +554,14 @@ TEST_P(WithMode, MultiWriteLarge) RecvBuf.Read(&ReadOffset, &BufferCount, ReadBuffers); ASSERT_FALSE(RecvBuf.HasUnreadData()); ASSERT_EQ(0ull, ReadOffset); - ASSERT_EQ(1u, BufferCount); - ASSERT_EQ(256u, ReadBuffers[0].Length); + if (Mode == QUIC_RECV_BUF_MODE_EXTERNAL) { + ASSERT_EQ(2u, BufferCount); + ASSERT_EQ(64u, ReadBuffers[0].Length); + ASSERT_EQ(192u, ReadBuffers[1].Length); + } else { + ASSERT_EQ(1u, BufferCount); + ASSERT_EQ(256u, ReadBuffers[0].Length); + } ASSERT_TRUE(RecvBuf.Drain(256)); ASSERT_FALSE(RecvBuf.HasUnreadData()); } @@ -569,17 +614,17 @@ TEST_P(WithMode, ReadPartial) ASSERT_EQ(2u, BufferCount); ASSERT_EQ(32u, ReadBuffers[0].Length); ASSERT_EQ(16u, ReadBuffers[1].Length); - } else { + } else if (Mode == QUIC_RECV_BUF_MODE_SINGLE) { ASSERT_EQ(16ull, ReadOffset); - if (Mode == QUIC_RECV_BUF_MODE_SINGLE) { - ASSERT_EQ(1u, BufferCount); - ASSERT_EQ(64u, ReadBuffers[0].Length); - } else { - ASSERT_EQ(2u, BufferCount); - ASSERT_EQ(48u, ReadBuffers[0].Length); - ASSERT_EQ(16u, ReadBuffers[1].Length); - } + ASSERT_EQ(1u, BufferCount); + ASSERT_EQ(64u, ReadBuffers[0].Length); + } else { // Mode == QUIC_RECV_BUF_MODE_CIRCULAR || QUIC_RECV_BUF_MODE_EXTERNAL + ASSERT_EQ(16ull, ReadOffset); + ASSERT_EQ(2u, BufferCount); + ASSERT_EQ(48u, ReadBuffers[0].Length); + ASSERT_EQ(16u, ReadBuffers[1].Length); } + ASSERT_TRUE(RecvBuf.Drain(64)); ASSERT_FALSE(RecvBuf.HasUnreadData()); } @@ -627,7 +672,8 @@ TEST_P(WithMode, ReadPendingMultiWrite) RecvBuf.Read(&ReadOffset, &BufferCount, ReadBuffers); ASSERT_FALSE(RecvBuf.HasUnreadData()); ASSERT_EQ(32ull, ReadOffset); - if (Mode == QUIC_RECV_BUF_MODE_MULTIPLE) { + if (Mode == QUIC_RECV_BUF_MODE_MULTIPLE || + Mode == QUIC_RECV_BUF_MODE_EXTERNAL) { ASSERT_EQ(2u, BufferCount); ASSERT_EQ(32u, ReadBuffers[0].Length); ASSERT_EQ(16u, ReadBuffers[1].Length); @@ -766,6 +812,129 @@ TEST_P(WithMode, DrainFrontChunkWithPendingGap) ASSERT_TRUE(RecvBuf.Drain(1)); } +TEST_P(WithMode, DrainFrontChunkWithPendingGap2) +{ + RecvBuffer RecvBuf; + auto Mode = GetParam(); + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.Initialize(Mode, false, 8, DEF_TEST_BUFFER_LENGTH)); + uint64_t InOutWriteLength = DEF_TEST_BUFFER_LENGTH; + BOOLEAN NewDataReady = FALSE; + + // Fill the first chunk partially + ASSERT_EQ(QUIC_STATUS_SUCCESS,RecvBuf.Write(0, 7, &InOutWriteLength, &NewDataReady)); + ASSERT_TRUE(RecvBuf.HasUnreadData()); + + // Read the data + uint64_t ReadOffset; + QUIC_BUFFER ReadBuffers[3]; + uint32_t BufferCount = ARRAYSIZE(ReadBuffers); + RecvBuf.Read(&ReadOffset, &BufferCount, ReadBuffers); + + // Before completing the read, write more non-ajacent data forcing the use of a new chunk + InOutWriteLength = DEF_TEST_BUFFER_LENGTH; + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.Write(9, 4, &InOutWriteLength, &NewDataReady)); + + RecvBuf.Drain(7); + + // After draining, ensure the front chunk was removed only for mode that + // copied the data to the new chunk + BOOLEAN ExternalReferences[] = {FALSE, FALSE}; + if (Mode == QUIC_RECV_BUF_MODE_SINGLE) { + RecvBuf.Check(0, 0, 1, ExternalReferences); + } else if (Mode == QUIC_RECV_BUF_MODE_CIRCULAR) { + RecvBuf.Check(7, 0, 1, ExternalReferences); + } else { + RecvBuf.Check(7, 0, 2, ExternalReferences); + } + + // Write to fill the gap and read all data + InOutWriteLength = DEF_TEST_BUFFER_LENGTH; + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.Write(7, 2, &InOutWriteLength, &NewDataReady)); + + BufferCount = ARRAYSIZE(ReadBuffers); + RecvBuf.Read(&ReadOffset, &BufferCount, ReadBuffers); + RecvBuf.Drain(6); +} + +TEST_P(WithMode, DrainFrontChunkExactly) +{ + RecvBuffer RecvBuf; + auto Mode = GetParam(); + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.Initialize(Mode, false, 8, DEF_TEST_BUFFER_LENGTH)); + uint64_t InOutWriteLength = DEF_TEST_BUFFER_LENGTH; + BOOLEAN NewDataReady = FALSE; + + // Fill the first chunk exactly + ASSERT_EQ(QUIC_STATUS_SUCCESS,RecvBuf.Write(0, 8, &InOutWriteLength, &NewDataReady)); + ASSERT_TRUE(RecvBuf.HasUnreadData()); + + // Read the data + uint64_t ReadOffset; + QUIC_BUFFER ReadBuffers[3]; + uint32_t BufferCount = ARRAYSIZE(ReadBuffers); + RecvBuf.Read(&ReadOffset, &BufferCount, ReadBuffers); + + // Before completing the read, write more non-ajacent data + InOutWriteLength = DEF_TEST_BUFFER_LENGTH; + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.Write(9, 4, &InOutWriteLength, &NewDataReady)); + + RecvBuf.Drain(8); + + // After draining, ensure the front chunk was properly removed + BOOLEAN ExternalReferences[] = {FALSE, FALSE}; + if (Mode == QUIC_RECV_BUF_MODE_CIRCULAR) { + RecvBuf.Check(8, 0, 1, ExternalReferences); + } else { + RecvBuf.Check(0, 0, 1, ExternalReferences); + } + + // Write to fill the gap and read all data + InOutWriteLength = DEF_TEST_BUFFER_LENGTH; + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.Write(8, 1, &InOutWriteLength, &NewDataReady)); + + BufferCount = ARRAYSIZE(ReadBuffers); + RecvBuf.Read(&ReadOffset, &BufferCount, ReadBuffers); + RecvBuf.Drain(5); +} + +TEST_P(WithMode, DrainFrontChunkExactly_NoGap) +{ + RecvBuffer RecvBuf; + auto Mode = GetParam(); + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.Initialize(Mode, false, 8, DEF_TEST_BUFFER_LENGTH)); + uint64_t InOutWriteLength = DEF_TEST_BUFFER_LENGTH; + BOOLEAN NewDataReady = FALSE; + + // Fill the first chunk exactly + ASSERT_EQ(QUIC_STATUS_SUCCESS,RecvBuf.Write(0, 8, &InOutWriteLength, &NewDataReady)); + ASSERT_TRUE(RecvBuf.HasUnreadData()); + + // Read the data + uint64_t ReadOffset; + QUIC_BUFFER ReadBuffers[3]; + uint32_t BufferCount = ARRAYSIZE(ReadBuffers); + RecvBuf.Read(&ReadOffset, &BufferCount, ReadBuffers); + + // Before completing the read, write more ajacent data + InOutWriteLength = DEF_TEST_BUFFER_LENGTH; + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.Write(8, 4, &InOutWriteLength, &NewDataReady)); + + RecvBuf.Drain(8); + + // After draining, ensure the front chunk was properly removed + BOOLEAN ExternalReferences[] = {FALSE, FALSE}; + if (Mode == QUIC_RECV_BUF_MODE_CIRCULAR) { + RecvBuf.Check(8, 4, 1, ExternalReferences); + } else { + RecvBuf.Check(0, 4, 1, ExternalReferences); + } + + // Read the rest of the data + BufferCount = ARRAYSIZE(ReadBuffers); + RecvBuf.Read(&ReadOffset, &BufferCount, ReadBuffers); + RecvBuf.Drain(4); +} + // Validate the gap can span the edge of a chunk // |0, 1, 2, 3, x, x, x, x| ReadStart:0, ReadLength:4, Ext:0 // |R, R, R, R, x, x, x, x| ReadStart:0, ReadLength:4, Ext:1 @@ -1451,8 +1620,240 @@ TEST(MultiRecvTest, ReadPendingOver2Chunk) RecvBuf.Drain(8); } +// +// Helper to build a list of external chunks +// +QUIC_STATUS MakeExternalChunks( + const std::vector& ChunkSizes, + size_t BufferSize, + _In_reads_bytes_(BufferSize) uint8_t* Buffer, + _Out_ CXPLAT_LIST_ENTRY* ChunkList) { + + uint64_t totalSize = 0; + for (auto size: ChunkSizes) { + totalSize += size; + } + if (totalSize > BufferSize) { + QUIC_STATUS_INVALID_PARAMETER; + } + + CxPlatListInitializeHead(ChunkList); + for (auto size: ChunkSizes) { + auto* chunk = reinterpret_cast(CXPLAT_ALLOC_NONPAGED(sizeof(QUIC_RECV_CHUNK), QUIC_POOL_RECVBUF)); + QuicRecvChunkInitialize(chunk, size, Buffer); + Buffer = Buffer + size; + CxPlatListInsertTail(ChunkList, &chunk->Link); + } + + return QUIC_STATUS_SUCCESS; +} + +void FreeChunkList(_Inout_ CXPLAT_LIST_ENTRY* ChunkList) { + while (!CxPlatListIsEmpty(ChunkList)) { + QUIC_RECV_CHUNK *Chunk = CXPLAT_CONTAINING_RECORD(CxPlatListRemoveHead(ChunkList), QUIC_RECV_CHUNK, Link); + CXPLAT_FREE(Chunk, QUIC_POOL_RECVBUF); + } +} + +TEST(ExternalBuffersTest, ProvideChunks) +{ + RecvBuffer RecvBuf; + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.Initialize(QUIC_RECV_BUF_MODE_EXTERNAL, false, 0, 0)); + + // + // Providing external chunks succeeds and change the buffer mode to external. + // + std::array Buffer{}; + std::vector ChunkSizes{8u, 8u}; + CXPLAT_LIST_ENTRY ChunkList; + ASSERT_EQ(QUIC_STATUS_SUCCESS, MakeExternalChunks(ChunkSizes, Buffer.size(), Buffer.data(), &ChunkList)); + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.ProvideChunks(&ChunkList)); + ASSERT_EQ(RecvBuf.RecvBuf.RecvMode, QUIC_RECV_BUF_MODE_EXTERNAL); + ASSERT_TRUE(CxPlatListIsEmpty(&ChunkList)); + + uint64_t InOutWriteLength = DEF_TEST_BUFFER_LENGTH; + BOOLEAN NewDataReady = FALSE; + RecvBuf.Write(0, 8, &InOutWriteLength, &NewDataReady); + + // + // More external buffers can be added, even after a write. + // + std::array Buffer2{}; + std::vector ChunkSizes2{8u, 8u}; + ASSERT_EQ(QUIC_STATUS_SUCCESS, MakeExternalChunks(ChunkSizes2, Buffer2.size(), Buffer2.data(), &ChunkList)); + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.ProvideChunks(&ChunkList)); + ASSERT_TRUE(CxPlatListIsEmpty(&ChunkList)); +} + +TEST(ExternalBuffersTest, ProvideChunksOverflow) +{ + RecvBuffer RecvBuf; + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.Initialize(QUIC_RECV_BUF_MODE_EXTERNAL, false, 0, 0)); + + // + // Ensure external buffers cannot be provided in a way that would overflow + // the virtual size. + // + std::array Buffer{}; + std::vector ChunkSizes{8u, 8u, 8u}; + CXPLAT_LIST_ENTRY ChunkList; + ASSERT_EQ(QUIC_STATUS_SUCCESS, MakeExternalChunks(ChunkSizes, Buffer.size(), Buffer.data(), &ChunkList)); + + for (CXPLAT_LIST_ENTRY* Entry = ChunkList.Flink; + Entry != &ChunkList; + Entry = Entry->Flink) { + auto* Chunk = CXPLAT_CONTAINING_RECORD(Entry, QUIC_RECV_CHUNK, Link); + // + // Lie about the actual size of the chunk, nobody will look at it. + // We don't want to allocate 4GB for real. + // + Chunk->AllocLength = 0x7000'0000; + } + + ASSERT_EQ(QUIC_STATUS_INVALID_PARAMETER, RecvBuf.ProvideChunks(&ChunkList)); + ASSERT_FALSE(CxPlatListIsEmpty(&ChunkList)); + + FreeChunkList(&ChunkList); +} + +TEST(ExternalBuffersTest, ReadWriteManyChunks) +{ + RecvBuffer RecvBuf; + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.Initialize(QUIC_RECV_BUF_MODE_EXTERNAL, false, 0, 0)); + + const uint32_t NbChunks = 5; + std::array Buffer{}; + std::vector ChunkSizes(NbChunks, 8u); + CXPLAT_LIST_ENTRY ChunkList; + ASSERT_EQ(QUIC_STATUS_SUCCESS, MakeExternalChunks(ChunkSizes, Buffer.size(), Buffer.data(), &ChunkList)); + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.ProvideChunks(&ChunkList)); + + std::vector ExternalReferences(NbChunks, FALSE); + RecvBuf.WriteAndCheck(10, 20, 0, 0, NbChunks, ExternalReferences.data()); + RecvBuf.WriteAndCheck(0, 10, 0, 8, NbChunks, ExternalReferences.data()); + + uint32_t LengthList[] = {8, 8, 8, 6}; + ExternalReferences[0] = TRUE; + ExternalReferences[1] = TRUE; + ExternalReferences[2] = TRUE; + ExternalReferences[3] = TRUE; + RecvBuf.ReadAndCheck(4, LengthList, 0, 8, NbChunks, ExternalReferences.data()); + RecvBuf.Drain(30); + + ExternalReferences[0] = FALSE; + ExternalReferences[1] = FALSE; + ExternalReferences[2] = FALSE; + ExternalReferences[3] = FALSE; + RecvBuf.Check(6, 0, 2, ExternalReferences.data()); +} + +TEST(ExternalBuffersTest, WriteTooLong) +{ + RecvBuffer RecvBuf; + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.Initialize(QUIC_RECV_BUF_MODE_EXTERNAL, false, 0, 0)); + + std::array Buffer{}; + std::vector ChunkSizes{8u, 8u}; + CXPLAT_LIST_ENTRY ChunkList; + ASSERT_EQ(QUIC_STATUS_SUCCESS, MakeExternalChunks(ChunkSizes, Buffer.size(), Buffer.data(), &ChunkList)); + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.ProvideChunks(&ChunkList)); + + uint64_t InOutWriteLength = DEF_TEST_BUFFER_LENGTH; + BOOLEAN NewDataReady = FALSE; + // + // Write 1 more byte than we have buffer space for. + // + ASSERT_EQ(QUIC_STATUS_BUFFER_TOO_SMALL, RecvBuf.Write(0, 17, &InOutWriteLength, &NewDataReady)); +} + +TEST(ExternalBuffersTest, OutOfBuffers) +{ + RecvBuffer RecvBuf; + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.Initialize(QUIC_RECV_BUF_MODE_EXTERNAL, false, 0, 0)); + + std::array Buffer{}; + std::vector ChunkSizes{DEF_TEST_BUFFER_LENGTH}; + CXPLAT_LIST_ENTRY ChunkList; + ASSERT_EQ(QUIC_STATUS_SUCCESS, MakeExternalChunks(ChunkSizes, Buffer.size(), Buffer.data(), &ChunkList)); + + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.ProvideChunks(&ChunkList)); + + uint64_t InOutWriteLength = DEF_TEST_BUFFER_LENGTH; + BOOLEAN NewDataReady = FALSE; + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.Write(0, DEF_TEST_BUFFER_LENGTH, &InOutWriteLength, &NewDataReady)); + + // + // Fully read and drain the only chunk, causing the chunk list to be empty. + // + uint32_t LengthList[] = {DEF_TEST_BUFFER_LENGTH}; + BOOLEAN ExternalReferences[] = {TRUE}; + RecvBuf.ReadAndCheck(1, LengthList, 0, DEF_TEST_BUFFER_LENGTH, 1, ExternalReferences); + + RecvBuf.Drain(DEF_TEST_BUFFER_LENGTH); + ExternalReferences[0] = FALSE; + RecvBuf.Check(0, 0, 0, ExternalReferences); + + // + // Make sure a write fail nicely in that state. + // + InOutWriteLength = DEF_TEST_BUFFER_LENGTH; + NewDataReady = FALSE; + ASSERT_EQ(QUIC_STATUS_BUFFER_TOO_SMALL, RecvBuf.Write(DEF_TEST_BUFFER_LENGTH, 8, &InOutWriteLength, &NewDataReady)); + + // + // Provide a new chunk and validate everything is back to normal. + // + ASSERT_EQ(QUIC_STATUS_SUCCESS, MakeExternalChunks(ChunkSizes, Buffer.size(), Buffer.data(), &ChunkList)); + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.ProvideChunks(&ChunkList)); + + InOutWriteLength = DEF_TEST_BUFFER_LENGTH; + NewDataReady = FALSE; + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.Write(DEF_TEST_BUFFER_LENGTH, 8, &InOutWriteLength, &NewDataReady)); + LengthList[0] = 8; + ExternalReferences[0] = TRUE; + RecvBuf.ReadAndCheck(1, LengthList, 0, 8, 1, ExternalReferences); + RecvBuf.Drain(8); +} + +TEST(ExternalBuffersTest, FreeBufferBeforeDrain) +{ + RecvBuffer RecvBuf; + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.Initialize(QUIC_RECV_BUF_MODE_EXTERNAL, false, 0, 0)); + + auto Buffer1 = std::make_unique(DEF_TEST_BUFFER_LENGTH); + std::array Buffer2{}; + std::vector ChunkSizes{DEF_TEST_BUFFER_LENGTH}; + CXPLAT_LIST_ENTRY ChunkList; + ASSERT_EQ(QUIC_STATUS_SUCCESS, MakeExternalChunks(ChunkSizes, DEF_TEST_BUFFER_LENGTH, Buffer1.get(), &ChunkList)); + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.ProvideChunks(&ChunkList)); + + ASSERT_EQ(QUIC_STATUS_SUCCESS, MakeExternalChunks(ChunkSizes, Buffer2.size(), Buffer2.data(), &ChunkList)); + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.ProvideChunks(&ChunkList)); + + uint64_t InOutWriteLength = DEF_TEST_BUFFER_LENGTH; + BOOLEAN NewDataReady = FALSE; + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.Write(0, DEF_TEST_BUFFER_LENGTH, &InOutWriteLength, &NewDataReady)); + uint32_t LengthList[] = {DEF_TEST_BUFFER_LENGTH}; + BOOLEAN ExternalReferences[] = {TRUE, FALSE}; + RecvBuf.ReadAndCheck(1, LengthList, 0, DEF_TEST_BUFFER_LENGTH, 2, ExternalReferences); + + // Free Buffer1 before draining. + Buffer1.reset(); + + RecvBuf.Drain(DEF_TEST_BUFFER_LENGTH); + + // Everything still good when writting and reading to the second chunk. + InOutWriteLength = DEF_TEST_BUFFER_LENGTH; + NewDataReady = FALSE; + ASSERT_EQ(QUIC_STATUS_SUCCESS, RecvBuf.Write(DEF_TEST_BUFFER_LENGTH, 8, &InOutWriteLength, &NewDataReady)); + LengthList[0] = 8; + ExternalReferences[0] = TRUE; + RecvBuf.ReadAndCheck(1, LengthList, 0, 8, 1, ExternalReferences); + RecvBuf.Drain(8); +} + INSTANTIATE_TEST_SUITE_P( RecvBufferTest, WithMode, - ::testing::Values(QUIC_RECV_BUF_MODE_SINGLE, QUIC_RECV_BUF_MODE_CIRCULAR, QUIC_RECV_BUF_MODE_MULTIPLE), + ::testing::Values(QUIC_RECV_BUF_MODE_SINGLE, QUIC_RECV_BUF_MODE_CIRCULAR, QUIC_RECV_BUF_MODE_MULTIPLE, QUIC_RECV_BUF_MODE_EXTERNAL), testing::PrintToStringParamName());