From b7e2b5081973990c15f0ff3f4969d667ab24261b Mon Sep 17 00:00:00 2001 From: feyris-tan <4116042+feyris-tan@users.noreply.github.com> Date: Tue, 21 Apr 2026 22:15:17 +0200 Subject: [PATCH] Got Reuters WNE file extraction working. --- Documentation/Reuters WNE Packet Format.md | 161 ++++--- skyscraper8.sln.DotSettings.user | 3 + skyscraper8/Properties/launchSettings.json | 2 +- skyscraper8/ReutersWne/ListByteArrayStream.cs | 177 +++++++ skyscraper8/ReutersWne/ReutersWneExtractor.cs | 439 ++++++++++++++++-- skyscraper8/ReutersWne/WneStory.cs | 74 +++ skyscraper8/Skyscraper/SpanByteExtensions.cs | 47 +- 7 files changed, 786 insertions(+), 117 deletions(-) create mode 100644 skyscraper8/ReutersWne/ListByteArrayStream.cs create mode 100644 skyscraper8/ReutersWne/WneStory.cs diff --git a/Documentation/Reuters WNE Packet Format.md b/Documentation/Reuters WNE Packet Format.md index d311ead..5c3e575 100644 --- a/Documentation/Reuters WNE Packet Format.md +++ b/Documentation/Reuters WNE Packet Format.md @@ -15,41 +15,44 @@ All multi-byte integers are little endian. ## About byte 1 Byte 1 determines what kind of packet we're dealing with. -| Byte 1 | Packet Type | -|--------|---------------------------------------------------------------------------------| -| 0x01 | A/V Payload, Metadata Payload, Metadata announcement, or Metadata closer packet | -| 0x03 | A/V announcement | -| 0xff | A/V closer | +| Byte 1 | Packet Type | +|--------|-----------------------------------------| +| 0x01 | Object transfer | +| 0x03 | Object announcement | +| 0xfe | Unknown, but seems to be related to ECC | +| 0xff | Object end marker | +## Packet familly 0x01 (Object transfer) +A packet of this type contains a single block of data. +| Byte Index | Description | +|------------|------------------------------| +| 0 | always 0x00 | +| 1 | always 0x01 | +| 2-3 | Packet length | +| 4-7 | Session ID | +| 8-11 | dependent on value of 12-15 | +| 12-13 | Payload description | +| 14-15 | Payload description argument | +| 16-end | Payload | +* When bytes 12-15 are all zeroes, the payload is raw data. In this case bytes 8-11 are the Block ID. To reconstruct the object, concatenate all bytes of Block ID 0, then Block ID 1, and so on... +* When byte 12 is 0x05, and byte 14 is 0x09, then the payload is prefixed with an additional 16-byte header. It's purpose is not yet known. The actual payload starts at byte 32 then. +* When the payload description argument equals the total packet length minus 15, then the payload is another encapsulated WNE packet. Start reading from byte 16 all over again. If this is the case, the following ECC rules apply: +### ECC rules +| Byte Index | Description | +|------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 8 | ECC group ID | +| 9 | ? | +| 10 | always 0x00 ? | +| 11 | ECC block counter, incremented by one each packet. If this is 0x07, the ECC group is finished, and the group ID will be incremented by one during the next packet. | +| 12 | When an ECC group is finished with byte 11 being 0x07, this contains the number of blocks that are supposed to be in this just finished ECC group. | +| 13 | usually 0x00, sometimes 0x20, purpose not known | +| 14-15 | If byte 12 is 0, then Packet length minus 15, If byte 12 is 0x07, then ? +| 16-end | If ecc flags is 0x07, then this is an ECC payload, if ECC flags is 0x00, continue reading. | -## Metadata Announcement packet structure (C) -| Byte Index | Description | -|------------|--------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0 | always 0x00 | -| 1 | always 0x01 | -| 2-3 | Packet length | -| 4-7 | Session ID | -| 8 | ECC group ID (shared with B/C) | -| 9 | ? | -| 10 | always 0x00 ? | -| 11 | ECC block counter, incremented by one each packet | -| 12 | ECC flags, if this is 0, the payload is raw data, if this is 0x07, the payload is ECC data and the next packet will have ECC group incremented by one. | -| 13 | usually 0x00, sometimes 0x20, purpose not known | -| 14-15 | If byte 12 is 0, then Packet length minus 15, If byte 12 is 0x07, then ? -| 16-end | If ecc flags is 0x07, then this is an ECC payload, if ECC flags is 0x00, continue reading. | -| Byte Index if ECC flags is 0 | Description | -|------------------------------|----------------------------------------------------------| -| 16 | Always 0x00 | -| 17 | Always 0x03 | -| 18-19 | Packet length minus 16 | -| 20-23 | File ID. This changes for every metadata file. | -| 24-28 | Block ID. Incremented by one for every part of the file. | -| 29-32 | Always 0x00000000 | -| 33-end | Payload | @@ -79,31 +82,7 @@ Byte 1 determines what kind of packet we're dealing with. -## Metadata Payload packet structure (B) -| Byte Index | Description | -|------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0 | always 0x00 | -| 1 | always 0x01 | -| 2-3 | Packet length | -| 4-7 | Session ID | -| 8 | ECC group ID (shared with B/C) | -| 9 | ? | -| 10 | always 0x00 ? | -| 11 | ECC block counter, incremented by one each packet | -| 12 | ECC flags, if this is 0, the payload is raw data, if this is 0x07, the payload is ECC data and the next packet will have ECC group incremented by one, and the block counter set to 0. | -| 13 | usually 0x00, sometimes 0x20, purpose not known | -| 14-15 | If byte 12 is 0, then Packet length minus 15, If byte 12 is 0x07, then ? | -| 16-end | If ecc flags is 0x07, then this is an ECC payload, if ECC flags is 0x00, continue reading. | -| Byte Index if ECC flags is 0 | Description | -|------------------------------|----------------------------------------------------------| -| 16 | Always 0x00 | -| 17 | Always 0x01 | -| 18-19 | Packet length minus 16 | -| 20-23 | File ID. This changes for every metadata file. | -| 24-28 | Block ID. Incremented by one for every part of the file. | -| 29-32 | Always 0x00000000 | -| 33-end | Payload | @@ -118,10 +97,47 @@ Byte 1 determines what kind of packet we're dealing with. +## Packet family 0x03 (Object announcement) +A packet of this type contains information about the file/object which is about to be transferred, or is transferred at this time in 0x01 packets. +| Byte Index | Description | +|------------|---------------------------------------------------------------| +| 0 | always 0x00 | +| 1 | always 0x03 | +| 2-3 | Packet length | +| 4-7 | Session ID | +| 8-11 | Packet length - 8 | +| 12-15 | Packet length - 12 | +| 16-19 | always seems to be 0x00000000 | +| 20-23 | always seems to be 0x10000000 | +| 24-27 | always seems to be 0x02000070 | +| 28-31 | always seems to be 0x00000000 | +| 32-35 | always seems to be 0x97120a00 | +| 36-39 | always seems to be 0x00000000 | +| 40 | specifies a subformat. Observed values here are 0x02 and 0x09 | +### When byte 40 is 0x09 +This indicates that the following object transfer does not contain a file, but a series of further encapsulated WNE packets. Those can be read using the same technique/structure. +### When byte 40 is 0x02 +This indicates that the following object transfer contains a file. +| Byte Index | Description | +|-------------|---------------------------------------------------| +| 44-45 | Packet length - 44 | +| 46-55 | unknown | +| 55-56 | size of each block (usually 1408, but may differ) | +| 57-63 | unknown | +| 64-67 | number of blocks in this object | +| 68-71 | unknown | +| 72-75 | also number of blocks in this object | +| 76-84 | unknown | +| 85-n | UTF-8 null terminated source file name | +| n+0 - n+54 | unknown | +| n+55 - n+59 | file length in bytes | +| n+60 - n+71 | unknown | +| n+72 - m | UTF-8 null terminated destination file name | +| m - end | unknown | @@ -131,15 +147,6 @@ Byte 1 determines what kind of packet we're dealing with. -## A/V Announcement packet structure (D) -| Byte Index | Description | -|------------|--------------------| -| 0 | always 0x00 | -| 1 | always 0x03 | -| 2-3 | Packet length | -| 4-7 | Session ID | -| 8-11 | Packet length - 8 | -| 12-15 | Packet length - 12 | @@ -170,11 +177,27 @@ Byte 1 determines what kind of packet we're dealing with. -## AV Payload packet structure (A) -| Byte Index | Description | -|------------|----------------| -| 0 | always 0x00 | -| 1 | always 0x01 | -| 2-3 | Packet length | -| 4-7 | Session ID | + + + + + + + + + + + + + + + + + + + +## Packet family 0xff (Object end marker) +When a packet of this type is encountered, the object transfer is complete, and the packets can be assembled. +These packets may appear multiple times, +Sometimes these packets are stuffed with multiple 0x00 bytes. diff --git a/skyscraper8.sln.DotSettings.user b/skyscraper8.sln.DotSettings.user index 5df9636..b14be1c 100644 --- a/skyscraper8.sln.DotSettings.user +++ b/skyscraper8.sln.DotSettings.user @@ -10,6 +10,7 @@ ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded ForceIncluded ForceIncluded @@ -27,8 +28,10 @@ ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded ForceIncluded ForceIncluded diff --git a/skyscraper8/Properties/launchSettings.json b/skyscraper8/Properties/launchSettings.json index 5a87411..76da5f0 100644 --- a/skyscraper8/Properties/launchSettings.json +++ b/skyscraper8/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "skyscraper8": { "commandName": "Project", - "commandLineArgs": "\"C:\\devel\\skyscraper8\\skyscraper8\\bin\\Debug\\net8.0\\reuters-dde-10g.ts\"", + "commandLineArgs": "\"F:\\2023_10_DVB-S\\0220W_SES4\\ses4_11126_h.ts\"", "remoteDebugEnabled": false }, "Container (Dockerfile)": { diff --git a/skyscraper8/ReutersWne/ListByteArrayStream.cs b/skyscraper8/ReutersWne/ListByteArrayStream.cs new file mode 100644 index 0000000..6c94a86 --- /dev/null +++ b/skyscraper8/ReutersWne/ListByteArrayStream.cs @@ -0,0 +1,177 @@ +using System; +using System.Collections.Generic; +using System.IO; + +public sealed class ListByteArrayStream : Stream +{ + private readonly IReadOnlyList _buffers; + private readonly long _sourceLength; + private long _position; + private long _length; + + public ListByteArrayStream(List buffers) + { + _buffers = buffers ?? throw new ArgumentNullException(nameof(buffers)); + + long length = 0; + for (int i = 0; i < _buffers.Count; i++) + length += _buffers[i]?.Length ?? 0; + + _sourceLength = length; + _length = length; + _position = 0; + } + + public override bool CanRead => true; + public override bool CanSeek => true; + public override bool CanWrite => false; + + public override long Length => _length; + + public override long Position + { + get => _position; + set + { + if (value < 0 || value > _length) + throw new ArgumentOutOfRangeException(nameof(value)); + + _position = value; + } + } + + public override int Read(byte[] buffer, int offset, int count) + { + ArgumentNullException.ThrowIfNull(buffer); + + if (offset < 0 || count < 0 || offset > buffer.Length - count) + throw new ArgumentOutOfRangeException(); + + if (count == 0 || _position >= _length) + return 0; + + int totalRead = 0; + + while (count > 0 && _position < _length) + { + LocatePosition(_position, out int bufferIndex, out int bufferOffset); + byte[] current = _buffers[bufferIndex] ?? Array.Empty(); + + long maxReadableInCurrentStream = Math.Min(current.Length - bufferOffset, _length - _position); + int toCopy = (int)Math.Min(count, maxReadableInCurrentStream); + + Buffer.BlockCopy(current, bufferOffset, buffer, offset, toCopy); + + offset += toCopy; + count -= toCopy; + totalRead += toCopy; + _position += toCopy; + } + + return totalRead; + } + + public override int Read(Span destination) + { + if (destination.IsEmpty || _position >= _length) + return 0; + + int totalRead = 0; + + while (!destination.IsEmpty && _position < _length) + { + LocatePosition(_position, out int bufferIndex, out int bufferOffset); + byte[] current = _buffers[bufferIndex] ?? Array.Empty(); + + long maxReadableInCurrentStream = Math.Min(current.Length - bufferOffset, _length - _position); + int toCopy = (int)Math.Min(destination.Length, maxReadableInCurrentStream); + + current.AsSpan(bufferOffset, toCopy).CopyTo(destination); + + destination = destination.Slice(toCopy); + totalRead += toCopy; + _position += toCopy; + } + + return totalRead; + } + + public override int ReadByte() + { + if (_position >= _length) + return -1; + + LocatePosition(_position, out int bufferIndex, out int bufferOffset); + byte[] current = _buffers[bufferIndex] ?? Array.Empty(); + + if (bufferOffset >= current.Length) + return -1; + + _position++; + return current[bufferOffset]; + } + + public override long Seek(long offset, SeekOrigin origin) + { + long newPosition = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => _position + offset, + SeekOrigin.End => _length + offset, + _ => throw new ArgumentOutOfRangeException(nameof(origin)) + }; + + if (newPosition < 0 || newPosition > _length) + throw new IOException("Attempted to seek outside the stream bounds."); + + _position = newPosition; + return _position; + } + + public override void Flush() + { + // No-op + } + + public override void SetLength(long value) + { + if (value < 0) + throw new ArgumentOutOfRangeException(nameof(value)); + + if (value > _sourceLength) + throw new NotSupportedException("Enlarging the stream is not supported."); + + _length = value; + + if (_position > _length) + _position = _length; + } + + public override void Write(byte[] buffer, int offset, int count) + => throw new NotSupportedException("Writing is not supported."); + + public override void Write(ReadOnlySpan buffer) + => throw new NotSupportedException("Writing is not supported."); + + private void LocatePosition(long position, out int bufferIndex, out int bufferOffset) + { + long remaining = position; + + for (int i = 0; i < _buffers.Count; i++) + { + int currentLength = _buffers[i]?.Length ?? 0; + + if (remaining < currentLength) + { + bufferIndex = i; + bufferOffset = (int)remaining; + return; + } + + remaining -= currentLength; + } + + bufferIndex = _buffers.Count - 1; + bufferOffset = 0; + } +} diff --git a/skyscraper8/ReutersWne/ReutersWneExtractor.cs b/skyscraper8/ReutersWne/ReutersWneExtractor.cs index 883969c..2fab337 100644 --- a/skyscraper8/ReutersWne/ReutersWneExtractor.cs +++ b/skyscraper8/ReutersWne/ReutersWneExtractor.cs @@ -44,72 +44,263 @@ public class ReutersWneExtractor : ISkyscraperMpePlugin } private DirectoryInfo outputDirectory; - private int packetSerial = 0; + private ulong packetSerial = 0; + + private const bool PACKET_DUMPING_ENABLED = false; + private const int PACKET_DUMP_START = 2825800; + private const int PACKET_DUMP_END = 2825900; + private void DumpPacketIfNecessary(Span udpPayload) + { + if (!PACKET_DUMPING_ENABLED) + return; + + if (packetSerial > PACKET_DUMP_START) + { + if (packetSerial < PACKET_DUMP_END) + { + string fname = string.Format("wne_dump/wne_{0:D4}.bin", packetSerial); + FileInfo fi = new FileInfo(fname); + fi.Directory.EnsureExists(); + File.WriteAllBytes(fname, udpPayload.ToArray()); + } + } + } public void HandlePacket(InternetHeader internetHeader, byte[] ipv4Packet) { - Span udpPayload = new Span(ipv4Packet,8,ipv4Packet.Length-8); + Span udpPayload = new Span(ipv4Packet,8,ipv4Packet.Length-8); + packetSerial++; + DumpPacketIfNecessary(udpPayload); + HandlePacket(udpPayload); + } - byte byte0 = udpPayload[0]; - if (byte0 != 0x00) - return; + private bool HandlePacket(Span udpPayload) + { + if (packetSerial == 2825816) + { + + } + + byte byte0 = udpPayload[0]; + if (byte0 != 0x00) + return false; - byte msgFamily = udpPayload[1]; - ushort length = udpPayload.ReadUInt16LittleEndian(2); - uint sessionId = udpPayload.ReadUInt32LittleEndian(4); + byte msgFamily = udpPayload[1]; + ushort length = udpPayload.ReadUInt16LittleEndian(2); + uint sessionId = udpPayload.ReadUInt32LittleEndian(4); - if (length != udpPayload.Length && !(length == 16 && udpPayload.Length == 18)) - { - return; - } + if (length != udpPayload.Length && !(length == 16 && udpPayload.Length == 18) && msgFamily != 0xff && msgFamily != 0xfe) + { + return false; + } + + switch (msgFamily) + { + case 0x01: + return ParsePacketType1(udpPayload); + case 0x03: + return ParsePacketType3(udpPayload); + case 0xfe: + return false; + case 0xff: + return ParsePacketType255(udpPayload); + default: + OnError("Unknown packet type {0:X2}", msgFamily); + return false; + } + } - switch (msgFamily) - { - case 0x01: - ParsePacketType1(udpPayload); - break; - default: - OnError("Unknown packet type {0:X2}", msgFamily); - return; - } + private bool ParsePacketType255(Span udpPayload) + { + byte byte0 = udpPayload[0]; + if (byte0 != 0x00) + return false; + + byte msgFamily = udpPayload[1]; + if (msgFamily != 0xff) + return false; + + ushort length = udpPayload.ReadUInt16LittleEndian(2); + if (udpPayload.Length != length) + { + if (!udpPayload.Slice(8).IsBlank()) + { + OnError("Unexpected packet length in a 0xff packet, got {0}, expected {2}.", udpPayload.Length, 8); + return false; + } + } + + uint sessionId = udpPayload.ReadUInt32LittleEndian(4); + + WneStory currentStory = null; + if (!wneStories.ContainsKey(sessionId)) + { + OnError("Missed WNE story announcement #{0}", sessionId); + return false; + } + else + { + currentStory = wneStories[sessionId]; + } - /*if (packetSerial < 1100) - { - - if (outputDirectory == null) - { - outputDirectory = new DirectoryInfo("wne_dump"); - outputDirectory.EnsureExists(); - } - string fname = string.Format("wne_dump/wne_{0:D4}.bin", packetSerial); - File.WriteAllBytes(fname, udpPayload.ToArray()); - packetSerial++; - }*/ + if (currentStory.Delivered) + { + return true; + } + + if (currentStory.Corrupted) + { + OnError("Story #{0} is corrupted and can not be recovered.", sessionId); + wneStories.Remove(sessionId); + return false; + } + + if (currentStory.IsEmpty()) + { + OnError("Story #{0} is empty and does not need to be recovered.", sessionId); + wneStories.Remove(sessionId); + return true; + } + + DeliverFile(currentStory); + currentStory.Dispose(); + return true; + } + + private void DeliverFile(WneStory story) + { + //_logger.InfoFormat("Attempting to deliver story #{0}", story.SessionId); + //_logger.InfoFormat("Expected number of blocks: {0}", story.ExpectedPayloadBlock); + + ListByteArrayStream listByteArrayStream = story.ToStream(); + listByteArrayStream.SetLength(story.FileSize); + //_logger.InfoFormat("Expected Stream length: {0} ({0:X8})", listByteArrayStream.Length); + + if (outputDirectory == null) + { + outputDirectory = new DirectoryInfo("wne_delivery"); + outputDirectory.EnsureExists(); + } + + string destinationFileName = story.DestinationFileName; + if (destinationFileName.StartsWith("\\")) + destinationFileName = destinationFileName.Substring(1); + + string outfileName = Path.Combine(outputDirectory.FullName, destinationFileName); + FileInfo outFileInfo = new FileInfo(outfileName); + outFileInfo.Directory.EnsureExists(); + + FileStream fileStream = outFileInfo.OpenWrite(); + listByteArrayStream.CopyTo(fileStream); + fileStream.Flush(); + fileStream.Close(); + } + + private bool ParsePacketType3(Span udpPayload) + { + byte byte0 = udpPayload[0]; + if (byte0 != 0x00) + return false; + + byte msgFamily = udpPayload[1]; + if (msgFamily != 0x03) + return false; + ushort length = udpPayload.ReadUInt16LittleEndian(2); + uint sessionId = udpPayload.ReadUInt32LittleEndian(4); + uint thirdUint = udpPayload.ReadUInt32LittleEndian(8); + uint fourthUint = udpPayload.ReadUInt32LittleEndian(12); + + if (thirdUint != udpPayload.Length - 8) + { + return false; + } + + if (fourthUint != udpPayload.Length - 24) + { + return false; + } + + WneStory currentStory = null; + if (!wneStories.ContainsKey(sessionId)) + { + currentStory = new WneStory(sessionId); + wneStories.Add(sessionId, currentStory); + _logger.InfoFormat("Found new WNE story #{0}", sessionId); + } + else + { + currentStory = wneStories[sessionId]; + } + + byte typeDiscriminator = udpPayload[40]; + switch (typeDiscriminator) + { + case 0x02: + ushort type2LengthCheck = udpPayload.ReadUInt16LittleEndian(44); + if (type2LengthCheck != udpPayload.Length - 44) + { + OnError("Malformed packet type discriminator 2."); + return false; + } + + bool filenameAlreadyKnown = !string.IsNullOrEmpty(currentStory.DestinationFileName); + + int nextOffset = -1; + currentStory.SourceFileName = udpPayload.ReadNullTerminatedUtf8String(84, out nextOffset); + + nextOffset += 55; + currentStory.FileSize = udpPayload.ReadUInt32LittleEndian(nextOffset); + nextOffset += 4; + nextOffset += 12; + + currentStory.DestinationFileName = udpPayload.ReadNullTerminatedUtf8String(nextOffset, out nextOffset); + + if (!filenameAlreadyKnown) + { + _logger.InfoFormat("Got file announcement for story #{0}: {1}", sessionId, currentStory.DestinationFileName); + } + + return true; + + case 0x09: + currentStory.FileDeliveryType = 0x09; + return true; + default: + OnError("Unknown packet type discriminator in a 0x03 packet {0:X2}", typeDiscriminator); + return false; + } } private bool ParsePacketType1(Span udpPayload) { + byte byte0 = udpPayload[0]; if (byte0 != 0x00) return false; - + byte msgFamily = udpPayload[1]; + if (msgFamily != 0x01) + return false; ushort length = udpPayload.ReadUInt16LittleEndian(2); uint sessionId = udpPayload.ReadUInt32LittleEndian(4); uint fourthUint = udpPayload.ReadUInt32LittleEndian(12); if (fourthUint == 0) { - if (length == 16) + if (length == 16 && udpPayload[11] == 0x07) { //Empty packet, likely to be used to announce stories. if (!wneStories.ContainsKey(sessionId)) { _logger.InfoFormat("Found new WNE story #{0}", sessionId); - WneStory newStory = new WneStory(); + WneStory newStory = new WneStory(sessionId); newStory.EccGroup = udpPayload[8]; if (udpPayload[11] == 0x07) + { newStory.ExpectedEccGroup = (byte)(newStory.EccGroup + 1); + newStory.ExpectedContinuityCounter = 0; + } + wneStories.Add(sessionId, newStory); return true; } @@ -122,6 +313,7 @@ public class ReutersWneExtractor : ISkyscraperMpePlugin if (udpPayload[11] == 0x07) { currentStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1); + currentStory.ExpectedContinuityCounter = 0; } return true; } @@ -129,14 +321,173 @@ public class ReutersWneExtractor : ISkyscraperMpePlugin else { //A/V Payload - OnError("A/V Payloads not supported yet."); - return false; + if (!wneStories.ContainsKey(sessionId)) + { + OnError("Missed the announcement for story #{0}. Data blocks of it are therefore not usable.", sessionId); + return false; + } + + if (fourthUint != 0) + { + OnError("Unexpected payload packet in story #{0}. Expected {1}, got {2}.", sessionId, 0, fourthUint); + return false; + } + uint thirdUint = udpPayload.ReadUInt32LittleEndian(8); + if (thirdUint != wneStories[sessionId].ExpectedPayloadBlock) + { + if (!wneStories[sessionId].Corrupted) + { + OnError("Expected payload block {0} but got {1}. This story ({2}) is incomplete and can not be recovered.", wneStories[sessionId].ExpectedPayloadBlock, thirdUint, sessionId); + wneStories[sessionId].Corrupted = true; + } + return false; + } + + wneStories[sessionId].AppendPayloadBlock(udpPayload.Slice(16)); + return true; } } else { - OnError("Non-A/V Payloads not supported yet."); - return false; + WneStory outerStory = null; + if (!wneStories.ContainsKey(sessionId)) + { + OnError("Missed announcement of story #{0}", sessionId); + return false; + } + outerStory = wneStories[sessionId]; + + bool payloadIsEcc = false; + if (udpPayload[12] == 0x07) + { + outerStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1); + outerStory.ExpectedContinuityCounter = 0; + payloadIsEcc = true; + } + + if (payloadIsEcc) + { + //OnError("ECC Payloads not supported yet."); + return false; + } + + int lengthCheckTolerance = 15; + bool hasAdditionalHeader = false; + if (udpPayload[12] == 0x04) + { + lengthCheckTolerance = 8; + hasAdditionalHeader = true; + } + + + ushort embeddedPacketLengthCheck = udpPayload.ReadUInt16LittleEndian(14); + if (embeddedPacketLengthCheck == length - lengthCheckTolerance) + { + + uint embeddedSessionId = udpPayload.ReadUInt32LittleEndian(20); + if (embeddedSessionId == 0) + embeddedSessionId = sessionId; + if (!wneStories.ContainsKey(embeddedSessionId)) + { + byte embeddedPacketType = udpPayload[17]; + if (embeddedPacketType == 0x03) + { + wneStories.Add(embeddedSessionId, new WneStory(embeddedSessionId)); + _logger.InfoFormat("Found new embedded WNE story #{0}", embeddedSessionId); + } + else + { + OnError("Missed announcement of embedded WNE story #{0}", embeddedSessionId); + return false; + } + } + + if (outerStory.timesSucessfullySynced == 0) + { + outerStory.ExpectedEccGroup = udpPayload[8]; + outerStory.ExpectedContinuityCounter = udpPayload[11] + 1; + outerStory.timesSucessfullySynced++; + } + else + { + if (outerStory.ExpectedEccGroup != udpPayload[8]) + { + OnError("Expected ECC group {0} but got {1}", outerStory.ExpectedEccGroup, udpPayload[8]); + return false; + } + + if (udpPayload[0x000b] == 0x07 && udpPayload[0x000c] == 0x03) + { + outerStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1); + outerStory.ExpectedContinuityCounter = 0; + return true; + } + + bool isFileFinished = udpPayload[18] == 0xff; + + if (udpPayload[11] != 0x07 || udpPayload[12] != 0x04) + { + if (isFileFinished) + { + if (udpPayload[12] != 0x01) + { + OnError("Expected Continuity Counter {0} but got {1}", + outerStory.ExpectedContinuityCounter, udpPayload[11]); + return false; + } + + outerStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1); + outerStory.ExpectedContinuityCounter = 0; + return true; + } + else + { + outerStory.ExpectedEccGroup = udpPayload[8]; + outerStory.ExpectedContinuityCounter = udpPayload[11] + 1; + } + } + else + { + outerStory.ExpectedEccGroup = udpPayload[8]; + outerStory.ExpectedContinuityCounter = udpPayload[11] + 1; + outerStory.timesSucessfullySynced++; + } + } + + if (udpPayload[11] == 0x07) + { + outerStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1); + outerStory.ExpectedContinuityCounter = 0; + } + + Span embeddedPayload = udpPayload.Slice(16); + return HandlePacket(embeddedPayload); + + } + else + { + if (udpPayload.Slice(16).IsBlank()) + { + return true; + } + + if (udpPayload[12] == 0x05 && udpPayload[14] == 0x09) + { + uint thirdUint = udpPayload.ReadUInt32LittleEndian(8); + if (thirdUint != wneStories[sessionId].ExpectedPayloadBlock) + { + OnError("Expected payload block {0} but got {1}. This story ({2}) is incomplete and can not be recovered.", wneStories[sessionId].ExpectedPayloadBlock, thirdUint, sessionId); + wneStories[sessionId].Corrupted = true; + return false; + } + + wneStories[sessionId].AppendPayloadBlock(udpPayload.Slice(16)); + return true; + } + + OnError("Unknown packet type in a 0x01 packet."); + return false; + } } } @@ -154,6 +505,7 @@ public class ReutersWneExtractor : ISkyscraperMpePlugin loggedErrors = new HashSet(); string unpacked = String.Format(message, args); + unpacked = String.Format("Packet #{0}: {1}", packetSerial, unpacked); if (!loggedErrors.Contains(unpacked)) { loggedErrors.Add(unpacked); @@ -161,10 +513,5 @@ public class ReutersWneExtractor : ISkyscraperMpePlugin } } - private class WneStory - { - public byte EccGroup; - public byte ExpectedEccGroup; - public int timesSucessfullySynced; - } + } diff --git a/skyscraper8/ReutersWne/WneStory.cs b/skyscraper8/ReutersWne/WneStory.cs new file mode 100644 index 0000000..e063242 --- /dev/null +++ b/skyscraper8/ReutersWne/WneStory.cs @@ -0,0 +1,74 @@ +namespace skyscraper8.ReutersWne; + +internal class WneStory : IDisposable +{ + public uint SessionId { get; } + + public WneStory(uint sessionId) + { + SessionId = sessionId; + } + + public byte EccGroup; + public byte ExpectedEccGroup; + public int timesSucessfullySynced; + public int FileDeliveryType; + public int ExpectedContinuityCounter; + public string SourceFileName; + public string DestinationFileName; + public uint ExpectedPayloadBlock; + + private List payloadBlocks; + public bool Corrupted; + public bool Delivered; + public uint FileSize; + + public void AppendPayloadBlock(Span slice) + { + if (payloadBlocks == null) + { + payloadBlocks = new List(); + } + payloadBlocks.Add(slice.ToArray()); + ExpectedPayloadBlock++; + } + + public bool IsEmpty() + { + if (payloadBlocks == null) + { + return true; + } + + if (payloadBlocks.Count == 0) + { + return true; + } + + for (int i = 0; i < payloadBlocks.Count; i++) + { + if (payloadBlocks[i].Length > 0) + { + return false; + } + } + + return true; + } + + public ListByteArrayStream ToStream() + { + if (Delivered) + { + throw new InvalidOperationException("Cannot convert a delivered story to a stream."); + } + return new ListByteArrayStream(payloadBlocks); + } + + public void Dispose() + { + payloadBlocks?.Clear(); + payloadBlocks = null; + Delivered = true; + } +} diff --git a/skyscraper8/Skyscraper/SpanByteExtensions.cs b/skyscraper8/Skyscraper/SpanByteExtensions.cs index 63a5ac8..d76f37b 100644 --- a/skyscraper8/Skyscraper/SpanByteExtensions.cs +++ b/skyscraper8/Skyscraper/SpanByteExtensions.cs @@ -1,4 +1,5 @@ using System; +using System.Text; public static class SpanByteExtensions { @@ -58,5 +59,49 @@ public static class SpanByteExtensions public static uint ReadUInt32BigEndian(this Span span, int offset) => ReadUInt32BigEndian((ReadOnlySpan)span, offset); - + public static string ReadNullTerminatedUtf8String(this ReadOnlySpan span, int offset) + { + if ((uint)offset >= (uint)span.Length) + throw new ArgumentOutOfRangeException(nameof(offset)); + + int nullIndex = span.Slice(offset).IndexOf((byte)0); + if (nullIndex < 0) + throw new InvalidOperationException("No null terminator found in span."); + + return Encoding.UTF8.GetString(span.Slice(offset, nullIndex)); + } + + public static string ReadNullTerminatedUtf8String(this ReadOnlySpan span, int offset, out int nextOffset) + { + if ((uint)offset >= (uint)span.Length) + throw new ArgumentOutOfRangeException(nameof(offset)); + + int nullIndex = span.Slice(offset).IndexOf((byte)0); + if (nullIndex < 0) + throw new InvalidOperationException("No null terminator found in span."); + + nextOffset = offset + nullIndex + 1; + return Encoding.UTF8.GetString(span.Slice(offset, nullIndex)); + } + + public static string ReadNullTerminatedUtf8String(this Span span, int offset) + => ReadNullTerminatedUtf8String((ReadOnlySpan)span, offset); + + public static string ReadNullTerminatedUtf8String(this Span span, int offset, out int nextOffset) + => ReadNullTerminatedUtf8String((ReadOnlySpan)span, offset, out nextOffset); + + public static bool IsBlank(this Span span) + { + if (span.Length == 0) + return true; + else + { + for (int i = 0; i < span.Length; i++) + { + if (span[i] != 0) + return false; + } + return true; + } + } }