diff --git a/src/Toimik.WarcProtocol/Records/ContinuationRecord.cs b/src/Toimik.WarcProtocol/Records/ContinuationRecord.cs index 588de88..ae12896 100644 --- a/src/Toimik.WarcProtocol/Records/ContinuationRecord.cs +++ b/src/Toimik.WarcProtocol/Records/ContinuationRecord.cs @@ -23,6 +23,8 @@ namespace Toimik.WarcProtocol; public class ContinuationRecord : Record { + public const string FieldForIdentifiedPayloadType = "warc-identified-payload-type"; + public const string FieldForInfoId = "warc-warcinfo-id"; public const string FieldForPayloadDigest = "warc-payload-digest"; @@ -51,6 +53,7 @@ public class ContinuationRecord : Record FieldForSegmentOriginId, FieldForSegmentNumber, FieldForSegmentTotalLength, + FieldForIdentifiedPayloadType, }; public ContinuationRecord( @@ -62,6 +65,7 @@ public ContinuationRecord( Uri segmentOriginId, int segmentNumber, int? segmentTotalLength = null, + string? identifiedPayloadType = null, string? truncatedReason = null, DigestFactory? digestFactory = null) : this( @@ -75,6 +79,7 @@ public ContinuationRecord( segmentOriginId, segmentNumber, segmentTotalLength, + identifiedPayloadType, truncatedReason, digestFactory) { @@ -91,6 +96,7 @@ public ContinuationRecord( Uri segmentOriginId, int segmentNumber, int? segmentTotalLength = null, + string? identifiedPayloadType = null, string? truncatedReason = null, DigestFactory? digestFactory = null) : base( @@ -106,6 +112,7 @@ public ContinuationRecord( // REMINDER: This is not auto-generated because it must be identical to the source's PayloadDigest = payloadDigest; + IdentifiedPayloadType ??= identifiedPayloadType; InfoId = infoId; TargetUri = targetUri; SegmentOriginId = segmentOriginId; @@ -127,6 +134,8 @@ internal ContinuationRecord( { } + public string? IdentifiedPayloadType { get; private set; } + public Uri? InfoId { get; private set; } public string? PayloadDigest { get; private set; } @@ -156,6 +165,10 @@ protected internal override void Set(string field, string value) { switch (field.ToLower()) { + case FieldForIdentifiedPayloadType: + IdentifiedPayloadType = value; + break; + case FieldForInfoId: InfoId = Utils.RemoveBracketsFromUri(value); break; @@ -205,6 +218,10 @@ protected internal override void Set(string field, string value) text = $"WARC-Date: {Utils.FormatDate(Date)}{WarcParser.CrLf}"; break; + case FieldForIdentifiedPayloadType: + text = ToString("WARC-Identified-Payload-Type", IdentifiedPayloadType); + break; + case FieldForInfoId: text = ToString("WARC-Warcinfo-ID", Utils.AddBracketsToUri(InfoId)); break; diff --git a/src/Toimik.WarcProtocol/Records/ConversionRecord.cs b/src/Toimik.WarcProtocol/Records/ConversionRecord.cs index 14077fe..babe891 100644 --- a/src/Toimik.WarcProtocol/Records/ConversionRecord.cs +++ b/src/Toimik.WarcProtocol/Records/ConversionRecord.cs @@ -64,6 +64,7 @@ public ConversionRecord( Uri infoId, Uri targetUri, string? payloadDigest = null, + string? identifiedPayloadType = null, Uri? refersTo = null, bool isSegmented = false, string? truncatedReason = null, @@ -78,6 +79,7 @@ public ConversionRecord( infoId, targetUri, payloadDigest, + identifiedPayloadType, refersTo, isSegmented, truncatedReason, @@ -95,6 +97,7 @@ public ConversionRecord( Uri infoId, Uri targetUri, string? payloadDigest = null, + string? identifiedPayloadType = null, Uri? refersTo = null, bool isSegmented = false, string? truncatedReason = null, @@ -113,6 +116,7 @@ public ConversionRecord( SetContentBlock(recordBlock, isParsed); PayloadDigest = payloadDigest; + IdentifiedPayloadType ??= identifiedPayloadType; if (recordBlock.Length > 0) { ContentType = contentType; @@ -170,19 +174,24 @@ internal override void SetContentBlock(byte[] contentBlock, bool isParsed = true { base.SetContentBlock(contentBlock, isParsed); RecordBlock = contentBlock; - IdentifiedPayloadType = PayloadTypeIdentifier.Identify(RecordBlock); + if (!isParsed) + { + IdentifiedPayloadType = PayloadTypeIdentifier.Identify(RecordBlock); + } } protected internal override void Set(string field, string value) { - // NOTE: FieldForIdentifiedPayloadType, if any, is ignored because it is supposed to be - // auto detected when the content block is set switch (field.ToLower()) { case FieldForContentType: ContentType = value; break; + case FieldForIdentifiedPayloadType: + IdentifiedPayloadType = value; + break; + case FieldForInfoId: InfoId = Utils.RemoveBracketsFromUri(value); break; diff --git a/src/Toimik.WarcProtocol/Records/Record.cs b/src/Toimik.WarcProtocol/Records/Record.cs index fb250cd..5a8fd54 100644 --- a/src/Toimik.WarcProtocol/Records/Record.cs +++ b/src/Toimik.WarcProtocol/Records/Record.cs @@ -102,8 +102,7 @@ internal static string ToString(string field, object? value) internal virtual void SetContentBlock(byte[] contentBlock, bool isParsed = true) { /* Depending on the record's type, a content block consists of a record block and / or a - * payload. If both exists, they are delimited by a consecutive pair of '\r\n' where the - * first pair is found at the end of a line and the other is on its own line. + * payload. The subclasses are responsible to detect those values. */ if (!isParsed) diff --git a/src/Toimik.WarcProtocol/Records/RequestRecord.cs b/src/Toimik.WarcProtocol/Records/RequestRecord.cs index 6d237bd..37c3d2e 100644 --- a/src/Toimik.WarcProtocol/Records/RequestRecord.cs +++ b/src/Toimik.WarcProtocol/Records/RequestRecord.cs @@ -66,6 +66,7 @@ public RequestRecord( Uri infoId, Uri targetUri, string? payloadDigest = null, + string? identifiedPayloadType = null, IPAddress? ipAddress = null, ISet? concurrentTos = null, string? truncatedReason = null, @@ -80,6 +81,7 @@ public RequestRecord( infoId, targetUri, payloadDigest, + identifiedPayloadType, ipAddress, concurrentTos, truncatedReason, @@ -97,6 +99,7 @@ public RequestRecord( Uri infoId, Uri targetUri, string? payloadDigest = null, + string? identifiedPayloadType = null, IPAddress? ipAddress = null, ISet? concurrentTos = null, string? truncatedReason = null, @@ -115,6 +118,7 @@ public RequestRecord( SetContentBlock(contentBlock, isParsed); PayloadDigest = payloadDigest; + IdentifiedPayloadType ??= identifiedPayloadType; if (contentBlock.Length > 0) { ContentType = contentType; @@ -176,22 +180,22 @@ internal override void SetContentBlock(byte[] contentBlock, bool isParsed = true if (index == -1) { RecordBlock = Encoding.UTF8.GetString(contentBlock); - Payload = Array.Empty(); } else { RecordBlock = Encoding.UTF8.GetString(contentBlock[0..index]); - Payload = contentBlock[(index + (WarcParser.CrLf.Length * 2))..]; + Payload = contentBlock[(index + PayloadTypeIdentifier.Delimiter.Length)..]; + if (!isParsed) + { + IdentifiedPayloadType = PayloadTypeIdentifier.Identify(Payload); + } } ContentBlock = contentBlock; - IdentifiedPayloadType = PayloadTypeIdentifier.Identify(Payload); } protected internal override void Set(string field, string value) { - // NOTE: FieldForIdentifiedPayloadType, if any, is ignored because it is supposed to be - // auto detected when the content block is set switch (field.ToLower()) { case FieldForConcurrentTo: @@ -202,6 +206,10 @@ protected internal override void Set(string field, string value) ContentType = value; break; + case FieldForIdentifiedPayloadType: + IdentifiedPayloadType = value; + break; + case FieldForInfoId: InfoId = Utils.RemoveBracketsFromUri(value); break; @@ -255,6 +263,10 @@ protected internal override void Set(string field, string value) text = $"WARC-Date: {Utils.FormatDate(Date)}{WarcParser.CrLf}"; break; + case FieldForIdentifiedPayloadType: + text = ToString("WARC-Identified-Payload-Type", IdentifiedPayloadType); + break; + case FieldForInfoId: text = ToString("WARC-Warcinfo-ID", Utils.AddBracketsToUri(InfoId)); break; diff --git a/src/Toimik.WarcProtocol/Records/ResourceRecord.cs b/src/Toimik.WarcProtocol/Records/ResourceRecord.cs index 10f25c9..b10ed27 100644 --- a/src/Toimik.WarcProtocol/Records/ResourceRecord.cs +++ b/src/Toimik.WarcProtocol/Records/ResourceRecord.cs @@ -68,6 +68,7 @@ public ResourceRecord( Uri infoId, Uri targetUri, string? payloadDigest = null, + string? identifiedPayloadType = null, IPAddress? ipAddress = null, ISet? concurrentTos = null, bool isSegmented = false, @@ -83,6 +84,7 @@ public ResourceRecord( infoId, targetUri, payloadDigest, + identifiedPayloadType, ipAddress, concurrentTos, isSegmented, @@ -101,6 +103,7 @@ public ResourceRecord( Uri infoId, Uri targetUri, string? payloadDigest = null, + string? identifiedPayloadType = null, IPAddress? ipAddress = null, ISet? concurrentTos = null, bool isSegmented = false, @@ -120,6 +123,7 @@ public ResourceRecord( SetContentBlock(recordBlock, isParsed); PayloadDigest = payloadDigest; + IdentifiedPayloadType ??= identifiedPayloadType; if (recordBlock.Length > 0) { ContentType = contentType; @@ -181,13 +185,14 @@ internal override void SetContentBlock(byte[] contentBlock, bool isParsed = true { base.SetContentBlock(contentBlock, isParsed); RecordBlock = contentBlock; - IdentifiedPayloadType = PayloadTypeIdentifier.Identify(RecordBlock); + if (!isParsed) + { + IdentifiedPayloadType = PayloadTypeIdentifier.Identify(RecordBlock); + } } protected internal override void Set(string field, string value) { - // NOTE: FieldForIdentifiedPayloadType, if any, is ignored because it is supposed to be - // auto detected when the content block is set switch (field.ToLower()) { case FieldForConcurrentTo: @@ -198,6 +203,10 @@ protected internal override void Set(string field, string value) ContentType = value; break; + case FieldForIdentifiedPayloadType: + IdentifiedPayloadType = value; + break; + case FieldForInfoId: InfoId = Utils.RemoveBracketsFromUri(value); break; diff --git a/src/Toimik.WarcProtocol/Records/ResponseRecord.cs b/src/Toimik.WarcProtocol/Records/ResponseRecord.cs index 832ed5f..66e3590 100644 --- a/src/Toimik.WarcProtocol/Records/ResponseRecord.cs +++ b/src/Toimik.WarcProtocol/Records/ResponseRecord.cs @@ -69,6 +69,7 @@ public ResponseRecord( Uri infoId, Uri targetUri, string? payloadDigest = null, + string? identifiedPayloadType = null, IPAddress? ipAddress = null, ISet? concurrentTos = null, bool isSegmented = false, @@ -84,6 +85,7 @@ public ResponseRecord( infoId, targetUri, payloadDigest, + identifiedPayloadType, ipAddress, concurrentTos, isSegmented, @@ -102,6 +104,7 @@ public ResponseRecord( Uri infoId, Uri targetUri, string? payloadDigest = null, + string? identifiedPayloadType = null, IPAddress? ipAddress = null, ISet? concurrentTos = null, bool isSegmented = false, @@ -121,6 +124,7 @@ public ResponseRecord( SetContentBlock(contentBlock, isParsed); PayloadDigest = payloadDigest; + IdentifiedPayloadType ??= identifiedPayloadType; if (contentBlock.Length > 0) { ContentType = contentType; @@ -195,14 +199,15 @@ internal override void SetContentBlock(byte[] contentBlock, bool isParsed = true { RecordBlock = Encoding.UTF8.GetString(contentBlock[0..index]); Payload = contentBlock[(index + PayloadTypeIdentifier.Delimiter.Length)..]; - IdentifiedPayloadType = PayloadTypeIdentifier.Identify(Payload); + if (!isParsed) + { + IdentifiedPayloadType = PayloadTypeIdentifier.Identify(Payload); + } } } protected internal override void Set(string field, string value) { - // NOTE: FieldForIdentifiedPayloadType, if any, is ignored because it is supposed to be - // auto detected when the content block is set switch (field.ToLower()) { case FieldForConcurrentTo: @@ -213,6 +218,10 @@ protected internal override void Set(string field, string value) ContentType = value; break; + case FieldForIdentifiedPayloadType: + IdentifiedPayloadType = value; + break; + case FieldForInfoId: InfoId = Utils.RemoveBracketsFromUri(value); break; diff --git a/src/Toimik.WarcProtocol/Toimik.WarcProtocol.csproj b/src/Toimik.WarcProtocol/Toimik.WarcProtocol.csproj index bdfde31..02fe8cc 100644 --- a/src/Toimik.WarcProtocol/Toimik.WarcProtocol.csproj +++ b/src/Toimik.WarcProtocol/Toimik.WarcProtocol.csproj @@ -4,9 +4,9 @@ net6.0 enable Toimik.WarcProtocol - 0.7.2 + 0.8.0 Nurhafiz - 0.7.2 + 0.8.0 true Toimik diff --git a/tests/Toimik.WarcProtocol.Tests/ConversionRecordTest.cs b/tests/Toimik.WarcProtocol.Tests/ConversionRecordTest.cs index 8ea09b3..26e06e1 100644 --- a/tests/Toimik.WarcProtocol.Tests/ConversionRecordTest.cs +++ b/tests/Toimik.WarcProtocol.Tests/ConversionRecordTest.cs @@ -6,6 +6,35 @@ public class ConversionRecordTest { + [Fact] + public void CreateWithCustomPayloadTypeIdentifierAndRecordBlockThatIsThePayload() + { + var record = new ConversionRecord( + DateTime.Now, + new SingleCrlfPayloadTypeIdentifier(), + recordBlock: Encoding.UTF8.GetBytes("foobar"), + contentType: "text/plain", + infoId: new Uri("urn:uuid:b92e8444-34cf-472f-a86e-07b7845ecc05"), + targetUri: new Uri("file://var/www/htdoc/robots.txt")); + + Assert.Equal(SingleCrlfPayloadTypeIdentifier.PayloadType, record.IdentifiedPayloadType); + } + + [Fact] + public void CreateWithRecordBlockThatIsThePayload() + { + var record = new ConversionRecord( + DateTime.Now, + new PayloadTypeIdentifier(), + recordBlock: Encoding.UTF8.GetBytes("foo"), + contentType: "text/plain", + infoId: Utils.CreateId(), + targetUri: new Uri("dns://example.com")); + + Assert.Null(record.PayloadDigest); + Assert.Null(record.IdentifiedPayloadType); + } + [Fact] public void WithContinuation() { @@ -17,6 +46,7 @@ public void WithContinuation() var targetUri = new Uri("http://www.example.com"); const string ContentType = "text/plain"; var recordBlock = "foo"; + var conversionRecord = new ConversionRecord( now, payloadTypeIdentifier, @@ -29,7 +59,8 @@ public void WithContinuation() Assert.Equal("1.1", conversionRecord.Version); Assert.NotNull(conversionRecord.Id); Assert.Equal(payloadTypeIdentifier, conversionRecord.PayloadTypeIdentifier); - Assert.Equal(recordBlock, Encoding.UTF8.GetString(conversionRecord.RecordBlock!)); + var actualRecordBlock = Encoding.UTF8.GetString(conversionRecord.RecordBlock!); + Assert.Equal(recordBlock, actualRecordBlock); Assert.Equal(ContentType, conversionRecord.ContentType); recordBlock = "bar"; @@ -45,24 +76,11 @@ public void WithContinuation() Assert.Equal("1.1", continuationRecord.Version); Assert.NotNull(continuationRecord.Id); Assert.Equal(now, continuationRecord.Date); - Assert.Equal(recordBlock, Encoding.UTF8.GetString(continuationRecord.RecordBlock!)); + actualRecordBlock = Encoding.UTF8.GetString(continuationRecord.RecordBlock!); + Assert.Equal(recordBlock, actualRecordBlock); Assert.Equal(payloadDigest, continuationRecord.PayloadDigest); Assert.Equal(targetUri, continuationRecord.TargetUri); Assert.Equal(infoId, continuationRecord.InfoId); Assert.Equal(2, continuationRecord.SegmentNumber); } - - [Fact] - public void WithoutPayloadDigest() - { - var conversionRecord = new ConversionRecord( - DateTime.Now, - new PayloadTypeIdentifier(), - Encoding.UTF8.GetBytes("foo"), - contentType: "text/plain", - infoId: Utils.CreateId(), - targetUri: new Uri("dns://example.com")); - - Assert.Null(conversionRecord.PayloadDigest); - } } \ No newline at end of file diff --git a/tests/Toimik.WarcProtocol.Tests/Data/Valid/1.1/misc/request_gemini_w_payload.warc b/tests/Toimik.WarcProtocol.Tests/Data/Valid/1.1/misc/request_gemini_w_payload.warc new file mode 100644 index 0000000..bdc691b --- /dev/null +++ b/tests/Toimik.WarcProtocol.Tests/Data/Valid/1.1/misc/request_gemini_w_payload.warc @@ -0,0 +1,13 @@ +WARC/1.1 +WARC-IP-Address: 174.138.124.161 +WARC-Type: request +WARC-Record-ID: +WARC-Target-URI: gemini://gemi.dev/why-gemini.gmi +WARC-Date: 2021-09-17T16:39:20.181428Z +WARC-Concurrent-To: +Content-Type: application/gemini; msgtype=request +Content-Length: 102 + +gemini://gemi.dev/why-gemini.gmi +# A test file that doesn't have a double CRLF anywhere. +Just A Test \ No newline at end of file diff --git a/tests/Toimik.WarcProtocol.Tests/Data/Valid/1.1/misc/request_gemini_wo_payload.warc b/tests/Toimik.WarcProtocol.Tests/Data/Valid/1.1/misc/request_gemini_wo_payload.warc new file mode 100644 index 0000000..f6608e1 --- /dev/null +++ b/tests/Toimik.WarcProtocol.Tests/Data/Valid/1.1/misc/request_gemini_wo_payload.warc @@ -0,0 +1,11 @@ +WARC/1.1 +WARC-IP-Address: 174.138.124.161 +WARC-Type: request +WARC-Record-ID: +WARC-Target-URI: gemini://gemi.dev/why-gemini.gmi +WARC-Date: 2021-09-17T16:39:20.181428Z +WARC-Concurrent-To: +Content-Type: application/gemini; msgtype=request +Content-Length: 32 + +gemini://gemi.dev/why-gemini.gmi \ No newline at end of file diff --git a/tests/Toimik.WarcProtocol.Tests/MetadataRecordTest.cs b/tests/Toimik.WarcProtocol.Tests/MetadataRecordTest.cs index 04b39ad..7af2341 100644 --- a/tests/Toimik.WarcProtocol.Tests/MetadataRecordTest.cs +++ b/tests/Toimik.WarcProtocol.Tests/MetadataRecordTest.cs @@ -6,7 +6,7 @@ public class MetadataRecordTest { [Fact] - public void InstantiateUsingConstructorWithFewerParameters() + public void CreateWithFewerParameters() { var now = DateTime.Now; var contentBlock = "foobar"; diff --git a/tests/Toimik.WarcProtocol.Tests/RequestRecordTest.cs b/tests/Toimik.WarcProtocol.Tests/RequestRecordTest.cs index 55e94b0..e298702 100644 --- a/tests/Toimik.WarcProtocol.Tests/RequestRecordTest.cs +++ b/tests/Toimik.WarcProtocol.Tests/RequestRecordTest.cs @@ -1,13 +1,55 @@ namespace Toimik.WarcProtocol.Tests; using System; +using System.IO; +using System.Linq; using System.Text; +using System.Threading.Tasks; using Xunit; public class RequestRecordTest { [Fact] - public void InstantiateUsingConstructorWithFewerParameters() + public void CreateWithCustomPayloadTypeIdentifierAndContentBlockThatHasNoPayload() + { + const string ExpectedRecordBlock = "gemini://gemi.dev/why-gemini.gmi"; + + var record = new RequestRecord( + DateTime.Now, + new SingleCrlfPayloadTypeIdentifier(), + contentBlock: Encoding.UTF8.GetBytes(ExpectedRecordBlock), + contentType: "application/gemini; msgtype=request", + infoId: new Uri("urn:uuid:1d0cf87c-b70a-4df6-9ff8-dd599494058d"), + targetUri: new Uri("gemini://gemi.dev/why-gemini.gmi")); + + Assert.Equal(ExpectedRecordBlock, record.RecordBlock); + Assert.Null(record.Payload); + Assert.Null(record.IdentifiedPayloadType); + } + + [Fact] + public void CreateWithCustomPayloadTypeIdentifierAndContentBlockThatHasPayload() + { + const string ExpectedRecordBlock = "gemini://gemi.dev/why-gemini.gmi"; + var expectedPayload = $"# A test file that doesn't have a double CRLF anywhere.{WarcParser.CrLf}Just A Test"; + var contentBlock = $"{ExpectedRecordBlock}{SingleCrlfPayloadTypeIdentifier.CreateDelimiterText(SingleCrlfPayloadTypeIdentifier.GeminiDelimiter)}{expectedPayload}"; + + var record = new RequestRecord( + DateTime.Now, + new SingleCrlfPayloadTypeIdentifier(), + contentBlock: Encoding.UTF8.GetBytes(contentBlock), + contentType: "application/gemini; msgtype=request", + infoId: new Uri("urn:uuid:1d0cf87c-b70a-4df6-9ff8-dd599494058d"), + targetUri: new Uri("gemini://gemi.dev/why-gemini.gmi")); + + Assert.Equal(ExpectedRecordBlock, record.RecordBlock); + var actualPayload = Encoding.UTF8.GetString(record.Payload!); + Assert.Equal(expectedPayload, actualPayload); + Assert.Equal(SingleCrlfPayloadTypeIdentifier.PayloadType, record.IdentifiedPayloadType); + } + + [Fact] + public void CreateWithFewerParameters() { var now = DateTime.Now; var payloadTypeIdentifier = new PayloadTypeIdentifier(); @@ -19,6 +61,7 @@ public void InstantiateUsingConstructorWithFewerParameters() const string ContentType = "application/http;msgtype=request"; var infoId = Utils.CreateId(); var targetUri = new Uri("http://www.example.com"); + var record = new RequestRecord( now, payloadTypeIdentifier, @@ -37,6 +80,44 @@ public void InstantiateUsingConstructorWithFewerParameters() Assert.Equal(ContentType, record.ContentType); Assert.Equal(infoId, record.InfoId); Assert.Equal(targetUri, record.TargetUri); - Assert.Equal(Payload, Encoding.UTF8.GetString(record.Payload!)); + var actualPayload = Encoding.UTF8.GetString(record.Payload!); + Assert.Equal(Payload, actualPayload); + } + + [Fact] + public async Task ParseWithCustomPayloadTypeIdentifierAndContentBlockThatHasNoPayload() + { + const string ExpectedRecordBlock = "gemini://gemi.dev/why-gemini.gmi"; + var recordFactory = new RecordFactory(payloadTypeIdentifier: new SingleCrlfPayloadTypeIdentifier()); + var parser = new WarcParser(recordFactory); + var path = $"{WarcParserTest.DirectoryForValidRecords}1.1{Path.DirectorySeparatorChar}misc{Path.DirectorySeparatorChar}request_gemini_wo_payload.warc"; + + var records = await parser.Parse(path).ToListAsync().ConfigureAwait(false); + var actualRecord = (RequestRecord)records[0]; + + Assert.Equal(ExpectedRecordBlock, actualRecord.RecordBlock); + Assert.Null(actualRecord.Payload); + Assert.Null(actualRecord.IdentifiedPayloadType); + } + + [Fact] + public async Task ParseWithCustomPayloadTypeIdentifierAndContentBlockThatHasPayload() + { + const string ExpectedRecordBlock = "gemini://gemi.dev/why-gemini.gmi"; + var expectedPayload = $"# A test file that doesn't have a double CRLF anywhere.{WarcParser.CrLf}Just A Test"; + var recordFactory = new RecordFactory(payloadTypeIdentifier: new SingleCrlfPayloadTypeIdentifier()); + var parser = new WarcParser(recordFactory); + var path = $"{WarcParserTest.DirectoryForValidRecords}1.1{Path.DirectorySeparatorChar}misc{Path.DirectorySeparatorChar}request_gemini_w_payload.warc"; + + var records = await parser.Parse(path).ToListAsync().ConfigureAwait(false); + var actualRecord = (RequestRecord)records[0]; + + Assert.Equal(ExpectedRecordBlock, actualRecord.RecordBlock); + var payload = actualRecord.Payload!; + var actualPayload = Encoding.UTF8.GetString(payload); + Assert.Equal(expectedPayload, actualPayload); + var identifiedPayloadType = actualRecord.PayloadTypeIdentifier.Identify(payload); + Assert.Equal(SingleCrlfPayloadTypeIdentifier.PayloadType, identifiedPayloadType); + Assert.Null(actualRecord.IdentifiedPayloadType); } } \ No newline at end of file diff --git a/tests/Toimik.WarcProtocol.Tests/ResourceRecordTest.cs b/tests/Toimik.WarcProtocol.Tests/ResourceRecordTest.cs index 4b47fa4..2351aa1 100644 --- a/tests/Toimik.WarcProtocol.Tests/ResourceRecordTest.cs +++ b/tests/Toimik.WarcProtocol.Tests/ResourceRecordTest.cs @@ -6,6 +6,35 @@ public class ResourceRecordTest { + [Fact] + public void CreateWithCustomPayloadTypeIdentifierAndRecordBlockThatIsThePayload() + { + var record = new ResourceRecord( + DateTime.Now, + new SingleCrlfPayloadTypeIdentifier(), + recordBlock: Encoding.UTF8.GetBytes("foo"), + contentType: "text/plain", + infoId: Utils.CreateId(), + targetUri: new Uri("dns://example.com")); + + Assert.Equal(SingleCrlfPayloadTypeIdentifier.PayloadType, record.IdentifiedPayloadType); + } + + [Fact] + public void CreateWithRecordBlockThatIsThePayload() + { + var record = new ResourceRecord( + DateTime.Now, + new PayloadTypeIdentifier(), + recordBlock: Encoding.UTF8.GetBytes("foo"), + contentType: "text/plain", + infoId: Utils.CreateId(), + targetUri: new Uri("dns://example.com")); + + Assert.Null(record.PayloadDigest); + Assert.Null(record.IdentifiedPayloadType); + } + [Fact] public void WithContinuation() { @@ -17,6 +46,7 @@ public void WithContinuation() var infoId = Utils.CreateId(); var targetUri = new Uri("http://www.example.com"); var recordBlock = "foo"; + var resourceRecord = new ResourceRecord( now, payloadTypeIdentifier, @@ -29,7 +59,8 @@ public void WithContinuation() Assert.Equal("1.1", resourceRecord.Version); Assert.NotNull(resourceRecord.Id); Assert.Equal(payloadTypeIdentifier, resourceRecord.PayloadTypeIdentifier); - Assert.Equal(recordBlock, Encoding.UTF8.GetString(resourceRecord.RecordBlock!)); + var actualRecordBlock = Encoding.UTF8.GetString(resourceRecord.RecordBlock!); + Assert.Equal(recordBlock, actualRecordBlock); Assert.Equal(ContentType, resourceRecord.ContentType); recordBlock = "bar"; @@ -45,24 +76,11 @@ public void WithContinuation() Assert.Equal("1.1", continuationRecord.Version); Assert.NotNull(continuationRecord.Id); Assert.Equal(now, continuationRecord.Date); - Assert.Equal(recordBlock, Encoding.UTF8.GetString(continuationRecord.RecordBlock!)); + actualRecordBlock = Encoding.UTF8.GetString(continuationRecord.RecordBlock!); + Assert.Equal(recordBlock, actualRecordBlock); Assert.Equal(payloadDigest, continuationRecord.PayloadDigest); Assert.Equal(targetUri, continuationRecord.TargetUri); Assert.Equal(infoId, continuationRecord.InfoId); Assert.Equal(2, continuationRecord.SegmentNumber); } - - [Fact] - public void WithoutPayloadDigest() - { - var resourceRecord = new ResourceRecord( - DateTime.Now, - new PayloadTypeIdentifier(), - Encoding.UTF8.GetBytes("foo"), - contentType: "text/plain", - infoId: Utils.CreateId(), - targetUri: new Uri("dns://example.com")); - - Assert.Null(resourceRecord.PayloadDigest); - } } \ No newline at end of file diff --git a/tests/Toimik.WarcProtocol.Tests/ResponseRecordTest.cs b/tests/Toimik.WarcProtocol.Tests/ResponseRecordTest.cs index 53a2662..56ff976 100644 --- a/tests/Toimik.WarcProtocol.Tests/ResponseRecordTest.cs +++ b/tests/Toimik.WarcProtocol.Tests/ResponseRecordTest.cs @@ -10,7 +10,44 @@ public class ResponseRecordTest { [Fact] - public void InstantiateUsingConstructorWithFewerParameters() + public void CreateWithCustomPayloadTypeIdentifierAndContentBlockWithoutPayload() + { + const string ExpectedRecordBlock = "20 text/gemini"; + var record = new ResponseRecord( + DateTime.Now, + new SingleCrlfPayloadTypeIdentifier(), + contentBlock: Encoding.UTF8.GetBytes(ExpectedRecordBlock), + contentType: "application/gemini; msgtype=response", + infoId: new Uri("urn:uuid:1d0cf87c-b70a-4df6-9ff8-dd599494058d"), + targetUri: new Uri("gemini://gemi.dev/why-gemini.gmi")); + + Assert.Equal(ExpectedRecordBlock, record.RecordBlock); + Assert.Null(record.Payload); + Assert.Null(record.IdentifiedPayloadType); + } + + [Fact] + public void CreateWithCustomPayloadTypeIdentifierAndContentBlockWithPayload() + { + const string ExpectedRecordBlock = "20 text/gemini"; + var expectedPayload = $"# A test file that doesn't have a double CRLF anywhere.{WarcParser.CrLf}Just A Test"; + var contentBlock = $"{ExpectedRecordBlock}{SingleCrlfPayloadTypeIdentifier.CreateDelimiterText(SingleCrlfPayloadTypeIdentifier.GeminiDelimiter)}{expectedPayload}"; + var record = new ResponseRecord( + DateTime.Now, + new SingleCrlfPayloadTypeIdentifier(), + contentBlock: Encoding.UTF8.GetBytes(contentBlock), + contentType: "application/gemini; msgtype=response", + infoId: new Uri("urn:uuid:1d0cf87c-b70a-4df6-9ff8-dd599494058d"), + targetUri: new Uri("gemini://gemi.dev/why-gemini.gmi")); + + Assert.Equal(ExpectedRecordBlock, record.RecordBlock); + var actualPayload = Encoding.UTF8.GetString(record.Payload!); + Assert.Equal(expectedPayload, actualPayload); + Assert.Equal(SingleCrlfPayloadTypeIdentifier.PayloadType, record.IdentifiedPayloadType); + } + + [Fact] + public void CreateWithFewerParameters() { var now = DateTime.Now; var payloadTypeIdentifier = new PayloadTypeIdentifier(); @@ -20,6 +57,7 @@ public void InstantiateUsingConstructorWithFewerParameters() const string ContentType = "application/http;msgtype=response"; var infoId = Utils.CreateId(); var targetUri = new Uri("http://www.example.com"); + var record = new ResponseRecord( now, payloadTypeIdentifier, @@ -42,7 +80,7 @@ public void InstantiateUsingConstructorWithFewerParameters() } [Fact] - public async Task ParseWithPayloadThatDoesNotExist() + public async Task ParseWithCustomPayloadTypeIdentifierAndContentBlockWithoutPayload() { const string ExpectedRecordBlock = "20 text/gemini"; var recordFactory = new RecordFactory(payloadTypeIdentifier: new SingleCrlfPayloadTypeIdentifier()); @@ -58,7 +96,7 @@ public async Task ParseWithPayloadThatDoesNotExist() } [Fact] - public async Task ParseWithPayloadThatExistButDelimitedByCustomDelimiter() + public async Task ParseWithCustomPayloadTypeIdentifierAndContentBlockWithPayload() { const string ExpectedRecordBlock = "20 text/gemini"; var expectedPayload = $"# A test file that doesn't have a double CRLF anywhere.{WarcParser.CrLf}Just A Test"; @@ -73,31 +111,8 @@ public async Task ParseWithPayloadThatExistButDelimitedByCustomDelimiter() var payload = actualRecord.Payload!; var actualPayload = Encoding.UTF8.GetString(payload); Assert.Equal(expectedPayload, actualPayload); - Assert.Equal(SingleCrlfPayloadTypeIdentifier.PayloadType, actualRecord.IdentifiedPayloadType); - Assert.Equal(SingleCrlfPayloadTypeIdentifier.PayloadType, actualRecord.PayloadTypeIdentifier.Identify(payload)); - } - - private class SingleCrlfPayloadTypeIdentifier : PayloadTypeIdentifier - { - public const string PayloadType = "foobar"; - - private static readonly int[] GeminiDelimiter = new int[] - { - WarcParser.CarriageReturn, - WarcParser.LineFeed, - }; - - public SingleCrlfPayloadTypeIdentifier() - : base(GeminiDelimiter) - { - } - - public override string? Identify(byte[] payload) - { - var type = payload.Length == 0 - ? null - : PayloadType; - return type; - } + var identifiedPayloadType = actualRecord.PayloadTypeIdentifier.Identify(payload); + Assert.Equal(SingleCrlfPayloadTypeIdentifier.PayloadType, identifiedPayloadType); + Assert.Null(actualRecord.IdentifiedPayloadType); } } \ No newline at end of file diff --git a/tests/Toimik.WarcProtocol.Tests/RevisitRecordTest.cs b/tests/Toimik.WarcProtocol.Tests/RevisitRecordTest.cs index fa671c2..51f460f 100644 --- a/tests/Toimik.WarcProtocol.Tests/RevisitRecordTest.cs +++ b/tests/Toimik.WarcProtocol.Tests/RevisitRecordTest.cs @@ -6,7 +6,7 @@ public class RevisitRecordTest { [Fact] - public void InstantiateUsingConstructorWithFewerParameters() + public void CreateWithFewerParameters() { var now = DateTime.Now; const string RecordBlock = "foobar"; @@ -14,6 +14,7 @@ public void InstantiateUsingConstructorWithFewerParameters() var infoId = Utils.CreateId(); var targetUri = new Uri("http://www.example.com"); var profile = new Uri("http://netpreserve.org/warc/1.1/revisit/identical-payload-digest"); + var record = new RevisitRecord( now, RecordBlock, diff --git a/tests/Toimik.WarcProtocol.Tests/SingleCrlfPayloadTypeIdentifier.cs b/tests/Toimik.WarcProtocol.Tests/SingleCrlfPayloadTypeIdentifier.cs new file mode 100644 index 0000000..2740022 --- /dev/null +++ b/tests/Toimik.WarcProtocol.Tests/SingleCrlfPayloadTypeIdentifier.cs @@ -0,0 +1,33 @@ +namespace Toimik.WarcProtocol.Tests; + +using System.Text; + +public class SingleCrlfPayloadTypeIdentifier : PayloadTypeIdentifier +{ + public const string PayloadType = "foobar"; + + public static readonly int[] GeminiDelimiter = new int[] + { + WarcParser.CarriageReturn, + WarcParser.LineFeed, + }; + + public SingleCrlfPayloadTypeIdentifier() + : base(GeminiDelimiter) + { + } + + public static string CreateDelimiterText(int[] delimiter) + { + var builder = new StringBuilder(); + foreach (int character in delimiter) + { + builder.Append((char)character); + } + + var text = builder.ToString(); + return text; + } + + public override string? Identify(byte[] payload) => PayloadType; +} \ No newline at end of file diff --git a/tests/Toimik.WarcProtocol.Tests/TestUtils.cs b/tests/Toimik.WarcProtocol.Tests/TestUtils.cs index a4d37fa..bd2b6c8 100644 --- a/tests/Toimik.WarcProtocol.Tests/TestUtils.cs +++ b/tests/Toimik.WarcProtocol.Tests/TestUtils.cs @@ -35,6 +35,8 @@ public static void AssertContinuationRecord( Assert.Equal(ContinuationRecord.TypeName, record.Type); Assert.Equal(new Uri("urn:uuid:b92e8444-34cf-472f-a86e-07b7845ecc05"), record.InfoId); Assert.Equal(83, record.ContentLength); + Assert.Equal("foobar", record.TruncatedReason); + Assert.Equal("text/dns", record.IdentifiedPayloadType); } public static void AssertConversionRecord( @@ -59,6 +61,8 @@ public static void AssertConversionRecord( Assert.Equal("text/plain", record.ContentType); var actualContentType = new ContentTypeIdentifier().Identify(record); Assert.Equal("application/octet-stream", actualContentType); + Assert.Equal("foobar", record.TruncatedReason); + Assert.Equal("text/plain", record.IdentifiedPayloadType); } public static void AssertMetadataRecord( @@ -83,6 +87,7 @@ public static void AssertMetadataRecord( Assert.Equal(62, record.ContentLength); var actualContentType = new ContentTypeIdentifier().Identify(record); Assert.Equal(record.ContentType, actualContentType); + Assert.Equal("foobar", record.TruncatedReason); } public static void AssertRequestRecord( @@ -107,6 +112,8 @@ public static void AssertRequestRecord( Assert.Equal(190, record.ContentLength); var actualContentType = new ContentTypeIdentifier().Identify(record); Assert.Equal(record.ContentType, actualContentType); + Assert.Equal("foobar", record.TruncatedReason); + Assert.Equal("foo/bar", record.IdentifiedPayloadType); } public static void AssertResourceRecord( @@ -133,6 +140,8 @@ public static void AssertResourceRecord( Assert.Equal("text/dns", record.ContentType); var actualContentType = new ContentTypeIdentifier().Identify(record); Assert.Equal("text/dns", actualContentType); + Assert.Equal("foobar", record.TruncatedReason); + Assert.Equal("text/plain", record.IdentifiedPayloadType); } public static void AssertResponseRecord( @@ -158,6 +167,8 @@ public static void AssertResponseRecord( Assert.Equal(1679, record.ContentLength); var actualContentType = new ContentTypeIdentifier().Identify(record); Assert.Equal(record.ContentType, actualContentType); + Assert.Equal("foobar", record.TruncatedReason); + Assert.Equal("text/html", record.IdentifiedPayloadType); } public static void AssertRevisitRecordForIdentical( @@ -186,6 +197,7 @@ public static void AssertRevisitRecordForIdentical( Assert.Equal("irrelevant/but-still-parsed-for-preservation", record.ContentType); var actualContentType = new ContentTypeIdentifier().Identify(record); Assert.Equal("application/octet-stream", actualContentType); + Assert.Equal("foobar", record.TruncatedReason); } public static void AssertRevisitRecordForUnmodified( @@ -214,6 +226,7 @@ public static void AssertRevisitRecordForUnmodified( Assert.Equal("message/http", record.ContentType); var actualContentType = new ContentTypeIdentifier().Identify(record); Assert.Equal("application/octet-stream", actualContentType); + Assert.Equal("foobar", record.TruncatedReason); } public static void AssertWarcinfoRecord(WarcinfoRecord record, bool isWithoutBlockDigest = false) @@ -231,6 +244,7 @@ public static void AssertWarcinfoRecord(WarcinfoRecord record, bool isWithoutBlo Assert.Equal(241, record.ContentLength); var actualContentType = new ContentTypeIdentifier().Identify(record); Assert.Equal(record.ContentType, actualContentType); + Assert.Equal("foobar", record.TruncatedReason); } [ExcludeFromCodeCoverage] @@ -307,10 +321,6 @@ public static async Task TestFile( case ConversionRecord.TypeName: var conversionRecord = (ConversionRecord)record; - - // NOTE: See remarks #1 - Assert.Null(conversionRecord.IdentifiedPayloadType); - AssertConversionRecord(conversionRecord, version: "1.1"); recordCounter++; break; @@ -322,7 +332,6 @@ public static async Task TestFile( case RequestRecord.TypeName: var requestRecord = (RequestRecord)record; - Assert.Null(requestRecord.IdentifiedPayloadType); AssertRequestRecord(requestRecord, version: "1.1"); recordCounter++; break; @@ -334,10 +343,6 @@ public static async Task TestFile( case ResponseRecord.TypeName: var responseRecord = (ResponseRecord)record; - - // NOTE: See remarks #1 - Assert.Null(responseRecord.IdentifiedPayloadType); - AssertResponseRecord(responseRecord, version: "1.1"); recordCounter++; break; diff --git a/tests/Toimik.WarcProtocol.Tests/Toimik.WarcProtocol.Tests.csproj b/tests/Toimik.WarcProtocol.Tests/Toimik.WarcProtocol.Tests.csproj index d4bb8c1..b85e667 100644 --- a/tests/Toimik.WarcProtocol.Tests/Toimik.WarcProtocol.Tests.csproj +++ b/tests/Toimik.WarcProtocol.Tests/Toimik.WarcProtocol.Tests.csproj @@ -112,9 +112,15 @@ PreserveNewest + + PreserveNewest + PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/tests/Toimik.WarcProtocol.Tests/WarcParserTest.cs b/tests/Toimik.WarcProtocol.Tests/WarcParserTest.cs index 0410d6e..50eaa27 100644 --- a/tests/Toimik.WarcProtocol.Tests/WarcParserTest.cs +++ b/tests/Toimik.WarcProtocol.Tests/WarcParserTest.cs @@ -395,6 +395,7 @@ public async Task RecordForContinuationThatIsUncompressed(string version, bool i actualRecord.SegmentOriginId!, actualRecord.SegmentNumber, actualRecord.SegmentTotalLength, + actualRecord.IdentifiedPayloadType, actualRecord.TruncatedReason, digestFactory); @@ -449,16 +450,13 @@ public async Task RecordForConversionThatIsUncompressed(string version, bool isW actualRecord.InfoId!, actualRecord.TargetUri!, actualRecord.PayloadDigest, + actualRecord.IdentifiedPayloadType, actualRecord.RefersTo, actualRecord.IsSegmented(), actualRecord.TruncatedReason, digestFactory); Assert.Equal(expectedRecord.RecordBlock!.Length, actualRecord.ContentLength); - Assert.Null(expectedRecord.IdentifiedPayloadType); - - // NOTE: See remarks #1 - Assert.Null(actualRecord.IdentifiedPayloadType); TestUtils.AssertConversionRecord( actualRecord, @@ -472,10 +470,7 @@ public async Task RecordForConversionThatIsUncompressed(string version, bool isW ((List)orderedFields).Remove(WarcProtocol.Record.FieldForBlockDigest); } - var fields = AssertHeaderAndToString( - actualRecord, - orderedFields, - hasIgnoredIdentifiedPayloadType: true); + var fields = AssertHeaderAndToString(actualRecord, orderedFields); fields.Sort(); var expectedHeader = expectedRecord.GetHeader(fields); @@ -566,14 +561,13 @@ public async Task RecordForRequestThatIsUncompressed(string version, bool isWith actualRecord.InfoId!, actualRecord.TargetUri!, actualRecord.PayloadDigest, + actualRecord.IdentifiedPayloadType, actualRecord.IpAddress, actualRecord.ConcurrentTos, actualRecord.TruncatedReason, digestFactory); Assert.Equal(expectedRecord.ContentBlock!.Length, actualRecord.ContentLength); - Assert.Null(expectedRecord.IdentifiedPayloadType); - Assert.Null(actualRecord.IdentifiedPayloadType); TestUtils.AssertRequestRecord( actualRecord, @@ -587,10 +581,7 @@ public async Task RecordForRequestThatIsUncompressed(string version, bool isWith ((List)orderedFields).Remove(WarcProtocol.Record.FieldForBlockDigest); } - var fields = AssertHeaderAndToString( - actualRecord, - orderedFields, - hasIgnoredIdentifiedPayloadType: true); + var fields = AssertHeaderAndToString(actualRecord, orderedFields); fields.Sort(); var expectedHeader = expectedRecord.GetHeader(fields); @@ -626,6 +617,7 @@ public async Task RecordForResourceThatIsUncompressed(string version, bool isWit actualRecord.InfoId!, actualRecord.TargetUri!, actualRecord.PayloadDigest, + actualRecord.IdentifiedPayloadType, actualRecord.IpAddress, actualRecord.ConcurrentTos, actualRecord.IsSegmented(), @@ -633,10 +625,6 @@ public async Task RecordForResourceThatIsUncompressed(string version, bool isWit digestFactory); Assert.Equal(expectedRecord.RecordBlock!.Length, actualRecord.ContentLength); - Assert.Null(actualRecord.IdentifiedPayloadType); - - // NOTE: See remarks #1 - Assert.Null(actualRecord.IdentifiedPayloadType); TestUtils.AssertResourceRecord( actualRecord, @@ -650,10 +638,7 @@ public async Task RecordForResourceThatIsUncompressed(string version, bool isWit ((List)orderedFields).Remove(WarcProtocol.Record.FieldForBlockDigest); } - var fields = AssertHeaderAndToString( - actualRecord, - orderedFields, - hasIgnoredIdentifiedPayloadType: true); + var fields = AssertHeaderAndToString(actualRecord, orderedFields); fields.Sort(); var expectedHeader = expectedRecord.GetHeader(fields); @@ -689,6 +674,7 @@ public async Task RecordForResponseThatIsUncompressed(string version, bool isWit actualRecord.InfoId!, actualRecord.TargetUri!, actualRecord.PayloadDigest, + actualRecord.IdentifiedPayloadType, actualRecord.IpAddress, actualRecord.ConcurrentTos, actualRecord.IsSegmented(), @@ -696,10 +682,6 @@ public async Task RecordForResponseThatIsUncompressed(string version, bool isWit digestFactory); Assert.Equal(expectedRecord.ContentBlock!.Length, actualRecord.ContentLength); - Assert.Null(expectedRecord.IdentifiedPayloadType); - - // NOTE: See remarks #1 - Assert.Null(actualRecord.IdentifiedPayloadType); TestUtils.AssertResponseRecord( actualRecord, @@ -713,10 +695,7 @@ public async Task RecordForResponseThatIsUncompressed(string version, bool isWit ((List)orderedFields).Remove(WarcProtocol.Record.FieldForBlockDigest); } - var fields = AssertHeaderAndToString( - actualRecord, - orderedFields, - hasIgnoredIdentifiedPayloadType: true); + var fields = AssertHeaderAndToString(actualRecord, orderedFields); fields.Sort(); var expectedHeader = expectedRecord.GetHeader(fields); @@ -892,21 +871,13 @@ public async Task RecordForWarcinfoThatIsUncompressed(bool isWithoutBlockDigest) Assert.Equal(expectedHeader, actualHeader); } - private static List AssertHeaderAndToString( - WarcProtocol.Record record, - IEnumerable defaultOrderedFields, - bool hasIgnoredIdentifiedPayloadType = false) + private static List AssertHeaderAndToString(WarcProtocol.Record record, IEnumerable defaultOrderedFields) { var expectedHeader = record.GetHeader(); var expectedHeaderTokens = expectedHeader.Split(WarcParser.CrLf, StringSplitOptions.RemoveEmptyEntries); var actualFields = new List(defaultOrderedFields); - // NOTE: WARC-Identified-Payload-Type, if any, is intentionally ignored by the parser - // because that value must be independently generated. As the feature is not - // implemented, the header will not include the field-value pair. - var headerDeclarationAndFieldCount = hasIgnoredIdentifiedPayloadType - ? actualFields.Count - : actualFields.Count + 1; + var headerDeclarationAndFieldCount = actualFields.Count + 1; Assert.Equal(expectedHeaderTokens.Length, headerDeclarationAndFieldCount); var actualHeader = record.GetHeader(actualFields); @@ -1008,8 +979,4 @@ public override Stream CreateDecompressStream(Stream stream) return new GZipInputStream(stream); } } -} - -// NOTE: (Remarks #1) Although the value is specified, the parser ignores it because the value must -// be independently identified using PayloadTypeIdentifier.Identify(...), which is not yet -// implemented \ No newline at end of file +} \ No newline at end of file diff --git a/tests/Toimik.WarcProtocol.Tests/WarcinfoRecordTest.cs b/tests/Toimik.WarcProtocol.Tests/WarcinfoRecordTest.cs index ecefd8f..7c8ab28 100644 --- a/tests/Toimik.WarcProtocol.Tests/WarcinfoRecordTest.cs +++ b/tests/Toimik.WarcProtocol.Tests/WarcinfoRecordTest.cs @@ -6,12 +6,13 @@ public class WarcinfoRecordTest { [Fact] - public void InstantiateUsingConstructorWithFewerParameters() + public void CreateWithFewerParameters() { var now = DateTime.Now; const string ContentBlock = "..."; const string ContentType = "application/warc-fields"; const string Filename = "filename.warc"; + var record = new WarcinfoRecord( now, ContentBlock,