Got Reuters WNE file extraction working.

Documentation/Reuters WNE Packet Format.md

@@ -16,40 +16,42 @@ All multi-byte integers are little endian.
 Byte 1 determines what kind of packet we're dealing with.
 
 | Byte 1 | Packet Type                             |
-|--------|---------------------------------------------------------------------------------|
+|--------|-----------------------------------------|
-| 0x01   | A/V Payload, Metadata Payload, Metadata announcement, or Metadata closer packet |
+| 0x01   | Object transfer                         |
-| 0x03   | A/V announcement                                                                |
+| 0x03   | Object announcement                     |
-| 0xff   | A/V closer                                                                      |
+| 0xfe   | Unknown, but seems to be related to ECC |
+| 0xff   | Object end marker                       |
 
+## Packet familly 0x01 (Object transfer)
+A packet of this type contains a single block of data.
 
-
-
-
-## Metadata Announcement packet structure (C)
 | Byte Index | Description                  |
-|------------|--------------------------------------------------------------------------------------------------------------------------------------------------------|
+|------------|------------------------------|
 | 0          | always 0x00                  |
 | 1          | always 0x01                  |
 | 2-3        | Packet length                |
 | 4-7        | Session ID                   |
-| 8          | ECC group ID (shared with B/C)                                                                                                                         |
+| 8-11       | dependent on value of 12-15  |
+| 12-13      | Payload description          | 
+| 14-15      | Payload description argument |
+| 16-end     | Payload                      |
+
+* When bytes 12-15 are all zeroes, the payload is raw data. In this case bytes 8-11 are the Block ID. To reconstruct the object, concatenate all bytes of Block ID 0, then Block ID 1, and so on...
+* When byte 12 is 0x05, and byte 14 is 0x09, then the payload is prefixed with an additional 16-byte header. It's purpose is not yet known. The actual payload starts at byte 32 then.
+* When the payload description argument equals the total packet length minus 15, then the payload is another encapsulated WNE packet. Start reading from byte 16 all over again. If this is the case, the following ECC rules apply:
+
+### ECC rules
+| Byte Index | Description                                                                                                                                                        |
+|------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 8          | ECC group ID                                                                                                                                                       |
 | 9          | ?                                                                                                                                                                  |
 | 10         | always 0x00 ?                                                                                                                                                      |
-| 11         | ECC block counter, incremented by one each packet                                                                                                      |
+| 11         | ECC block counter, incremented by one each packet. If this is 0x07, the ECC group is finished, and the group ID will be incremented by one during the next packet. |
-| 12         | ECC flags, if this is 0, the payload is raw data, if this is 0x07, the payload is ECC data and the next packet will have ECC group incremented by one. |
+| 12         | When an ECC group is finished with byte 11 being 0x07, this contains the number of blocks that are supposed to be in this just finished ECC group.                 |
 | 13         | usually 0x00, sometimes 0x20, purpose not known                                                                                                                    |
 | 14-15      | If byte 12 is 0, then  Packet length minus 15, If byte 12 is 0x07, then ?                                                                                          
 | 16-end     | If ecc flags is 0x07, then this is an ECC payload, if ECC flags is 0x00, continue reading.                                                                         |
 
-| Byte Index if ECC flags is 0 | Description                                              |
-|------------------------------|----------------------------------------------------------|
-| 16                           | Always 0x00                                              |
-| 17                           | Always 0x03                                              |
-| 18-19                        | Packet length minus 16                                   |
-| 20-23                        | File ID. This changes for every metadata file.           |
-| 24-28                        | Block ID. Incremented by one for every part of the file. |
-| 29-32                        | Always 0x00000000                                        |
-| 33-end                       | Payload                                                  |
 
 
 
@@ -79,102 +81,123 @@ Byte 1 determines what kind of packet we're dealing with.
 
 
 
-## Metadata Payload packet structure (B)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+## Packet family 0x03 (Object announcement)
+A packet of this type contains information about the file/object which is about to be transferred, or is transferred at this time in 0x01 packets.
+
 | Byte Index | Description                                                   |
-|------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|------------|---------------------------------------------------------------|
-| 0          | always 0x00                                                                                                                                                                            |
-| 1          | always 0x01                                                                                                                                                                            |
-| 2-3        | Packet length                                                                                                                                                                          |
-| 4-7        | Session ID                                                                                                                                                                             |
-| 8          | ECC group ID (shared with B/C)                                                                                                                                                         |
-| 9          | ?                                                                                                                                                                                      |
-| 10         | always 0x00 ?                                                                                                                                                                          |
-| 11         | ECC block counter, incremented by one each packet                                                                                                                                      |
-| 12         | ECC flags, if this is 0, the payload is raw data, if this is 0x07, the payload is ECC data and the next packet will have ECC group incremented by one, and the block counter set to 0. |
-| 13         | usually 0x00, sometimes 0x20, purpose not known                                                                                                        |
-| 14-15      | If byte 12 is 0, then  Packet length minus 15, If byte 12 is 0x07, then ?                                                                                                              |
-| 16-end     | If ecc flags is 0x07, then this is an ECC payload, if ECC flags is 0x00, continue reading.                                                                                             |
-
-| Byte Index if ECC flags is 0 | Description                                              |
-|------------------------------|----------------------------------------------------------|
-| 16                           | Always 0x00                                              |
-| 17                           | Always 0x01                                              |
-| 18-19                        | Packet length minus 16                                   |
-| 20-23                        | File ID. This changes for every metadata file.           |
-| 24-28                        | Block ID. Incremented by one for every part of the file. |
-| 29-32                        | Always 0x00000000                                        |
-| 33-end                       | Payload                                                  |
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-## A/V Announcement packet structure (D)
-| Byte Index | Description        |
-|------------|--------------------|
 | 0          | always 0x00                                                   |
 | 1          | always 0x03                                                   |
 | 2-3        | Packet length                                                 |
 | 4-7        | Session ID                                                    |
 | 8-11       | Packet length - 8                                             |
 | 12-15      | Packet length - 12                                            |
+| 16-19      | always seems to be 0x00000000                                 |
+| 20-23      | always seems to be 0x10000000                                 |
+| 24-27      | always seems to be 0x02000070                                 |
+| 28-31      | always seems to be 0x00000000                                 |
+| 32-35      | always seems to be 0x97120a00                                 |
+| 36-39      | always seems to be 0x00000000                                 |
+| 40         | specifies a subformat. Observed values here are 0x02 and 0x09 |
 
+### When byte 40 is 0x09
+This indicates that the following object transfer does not contain a file, but a series of further encapsulated WNE packets. Those can be read using the same technique/structure. 
 
-
+### When byte 40 is 0x02
-
+This indicates that the following object transfer contains a file.
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-## AV Payload packet structure (A)
 
 | Byte Index  | Description                                       |
-|------------|----------------|
+|-------------|---------------------------------------------------|
-| 0          | always 0x00    |
+| 44-45       | Packet length - 44                                |
-| 1          | always 0x01    |
+| 46-55       | unknown                                           |
-| 2-3        | Packet length  |
+| 55-56       | size of each block (usually 1408, but may differ) |
-| 4-7        | Session ID     |
+| 57-63       | unknown                                           |
+| 64-67       | number of blocks in this object                   |
+| 68-71       | unknown                                           |
+| 72-75       | also number of blocks in this object              |
+| 76-84       | unknown                                           |
+| 85-n        | UTF-8 null terminated source file name            |
+| n+0 - n+54  | unknown                                           |
+| n+55 - n+59 | file length in bytes                              |
+| n+60 - n+71 | unknown                                           |
+| n+72 - m    | UTF-8 null terminated destination file name       |
+| m - end     | unknown                                           |
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+## Packet family 0xff (Object end marker)
+When a packet of this type is encountered, the object transfer is complete, and the packets can be assembled. 
+These packets may appear multiple times,
+Sometimes these packets are stuffed with multiple 0x00 bytes.

skyscraper8.sln.DotSettings.user

@@ -10,6 +10,7 @@
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AAssert_002EThrowsException_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FSourcesCache_003Fe6930caf46f7fcaa445fdd26a3884170a45a4766298259e7a29bae560adef_003FAssert_002EThrowsException_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ACultureInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F8e_003Fe58e5b4a_003FCultureInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADictionary_00602_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F83_003Fc73c45bc_003FDictionary_00602_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADictionary_00602_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2026_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Ffd0d7846a16a4147ad767b7c3ae2cf57b2e200_003F4a_003Ff8de026f_003FDictionary_00602_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFastResourceComparer_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F8f_003Ffb4ed86c_003FFastResourceComparer_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFileInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fe1ab690537c44e02a014076312b886b7b2e200_003F5a_003Fcf76af61_003FFileInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AInterop_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003Fea_003F7d70064b_003FInterop_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
@@ -27,8 +28,10 @@
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARuntimeInformation_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F4c_003F5f771d10_003FRuntimeInformation_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARuntimeResourceSet_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003Fe1_003Ff15f6bbe_003FRuntimeResourceSet_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASafeFileHandle_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003Fc6_003Fd8e0f2f2_003FSafeFileHandle_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASafeFileHandle_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2026_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Ffd0d7846a16a4147ad767b7c3ae2cf57b2e200_003F95_003Ff0feb47b_003FSafeFileHandle_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASkip_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FSourcesCache_003Fd8c543d93f1559af2ea2be8e9d55839b5bb1a3605f22daa45ea63772e3b4bc_003FSkip_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASpan_00601_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2026_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F48f7bf031b2942d3831d66d46fe3bc99b2e200_003F49_003F017cad58_003FSpan_00601_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASpan_00601_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2026_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Ffd0d7846a16a4147ad767b7c3ae2cf57b2e200_003F35_003F161711de_003FSpan_00601_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASR_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F4d_003F7edc51d9_003FSR_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F57d616db882b441b8c50720b4477e03db2e200_003F2e_003F1a14f40f_003FString_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATestMethodInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FSourcesCache_003F9ea467e7c7b4671a214143293021e7ec337916b71125d896e17a0b7fc_003FTestMethodInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>

skyscraper8/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "skyscraper8": {
       "commandName": "Project",
-      "commandLineArgs": "\"C:\\devel\\skyscraper8\\skyscraper8\\bin\\Debug\\net8.0\\reuters-dde-10g.ts\"",
+      "commandLineArgs": "\"F:\\2023_10_DVB-S\\0220W_SES4\\ses4_11126_h.ts\"",
       "remoteDebugEnabled": false
     },
     "Container (Dockerfile)": {

skyscraper8/ReutersWne/ListByteArrayStream.cs

@@ -0,0 +1,177 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+
+public sealed class ListByteArrayStream : Stream
+{
+    private readonly IReadOnlyList<byte[]> _buffers;
+    private readonly long _sourceLength;
+    private long _position;
+    private long _length;
+
+    public ListByteArrayStream(List<byte[]> buffers)
+    {
+        _buffers = buffers ?? throw new ArgumentNullException(nameof(buffers));
+
+        long length = 0;
+        for (int i = 0; i < _buffers.Count; i++)
+            length += _buffers[i]?.Length ?? 0;
+
+        _sourceLength = length;
+        _length = length;
+        _position = 0;
+    }
+
+    public override bool CanRead => true;
+    public override bool CanSeek => true;
+    public override bool CanWrite => false;
+
+    public override long Length => _length;
+
+    public override long Position
+    {
+        get => _position;
+        set
+        {
+            if (value < 0 || value > _length)
+                throw new ArgumentOutOfRangeException(nameof(value));
+
+            _position = value;
+        }
+    }
+
+    public override int Read(byte[] buffer, int offset, int count)
+    {
+        ArgumentNullException.ThrowIfNull(buffer);
+
+        if (offset < 0 || count < 0 || offset > buffer.Length - count)
+            throw new ArgumentOutOfRangeException();
+
+        if (count == 0 || _position >= _length)
+            return 0;
+
+        int totalRead = 0;
+
+        while (count > 0 && _position < _length)
+        {
+            LocatePosition(_position, out int bufferIndex, out int bufferOffset);
+            byte[] current = _buffers[bufferIndex] ?? Array.Empty<byte>();
+
+            long maxReadableInCurrentStream = Math.Min(current.Length - bufferOffset, _length - _position);
+            int toCopy = (int)Math.Min(count, maxReadableInCurrentStream);
+
+            Buffer.BlockCopy(current, bufferOffset, buffer, offset, toCopy);
+
+            offset += toCopy;
+            count -= toCopy;
+            totalRead += toCopy;
+            _position += toCopy;
+        }
+
+        return totalRead;
+    }
+
+    public override int Read(Span<byte> destination)
+    {
+        if (destination.IsEmpty || _position >= _length)
+            return 0;
+
+        int totalRead = 0;
+
+        while (!destination.IsEmpty && _position < _length)
+        {
+            LocatePosition(_position, out int bufferIndex, out int bufferOffset);
+            byte[] current = _buffers[bufferIndex] ?? Array.Empty<byte>();
+
+            long maxReadableInCurrentStream = Math.Min(current.Length - bufferOffset, _length - _position);
+            int toCopy = (int)Math.Min(destination.Length, maxReadableInCurrentStream);
+
+            current.AsSpan(bufferOffset, toCopy).CopyTo(destination);
+
+            destination = destination.Slice(toCopy);
+            totalRead += toCopy;
+            _position += toCopy;
+        }
+
+        return totalRead;
+    }
+
+    public override int ReadByte()
+    {
+        if (_position >= _length)
+            return -1;
+
+        LocatePosition(_position, out int bufferIndex, out int bufferOffset);
+        byte[] current = _buffers[bufferIndex] ?? Array.Empty<byte>();
+
+        if (bufferOffset >= current.Length)
+            return -1;
+
+        _position++;
+        return current[bufferOffset];
+    }
+
+    public override long Seek(long offset, SeekOrigin origin)
+    {
+        long newPosition = origin switch
+        {
+            SeekOrigin.Begin => offset,
+            SeekOrigin.Current => _position + offset,
+            SeekOrigin.End => _length + offset,
+            _ => throw new ArgumentOutOfRangeException(nameof(origin))
+        };
+
+        if (newPosition < 0 || newPosition > _length)
+            throw new IOException("Attempted to seek outside the stream bounds.");
+
+        _position = newPosition;
+        return _position;
+    }
+
+    public override void Flush()
+    {
+        // No-op
+    }
+
+    public override void SetLength(long value)
+    {
+        if (value < 0)
+            throw new ArgumentOutOfRangeException(nameof(value));
+
+        if (value > _sourceLength)
+            throw new NotSupportedException("Enlarging the stream is not supported.");
+
+        _length = value;
+
+        if (_position > _length)
+            _position = _length;
+    }
+
+    public override void Write(byte[] buffer, int offset, int count)
+        => throw new NotSupportedException("Writing is not supported.");
+
+    public override void Write(ReadOnlySpan<byte> buffer)
+        => throw new NotSupportedException("Writing is not supported.");
+
+    private void LocatePosition(long position, out int bufferIndex, out int bufferOffset)
+    {
+        long remaining = position;
+
+        for (int i = 0; i < _buffers.Count; i++)
+        {
+            int currentLength = _buffers[i]?.Length ?? 0;
+
+            if (remaining < currentLength)
+            {
+                bufferIndex = i;
+                bufferOffset = (int)remaining;
+                return;
+            }
+
+            remaining -= currentLength;
+        }
+
+        bufferIndex = _buffers.Count - 1;
+        bufferOffset = 0;
+    }
+}

skyscraper8/ReutersWne/ReutersWneExtractor.cs

@@ -44,51 +44,43 @@ public class ReutersWneExtractor : ISkyscraperMpePlugin
     }
     
     private DirectoryInfo outputDirectory;
-    private int packetSerial = 0;
+    private ulong packetSerial = 0;
+
+    private const bool PACKET_DUMPING_ENABLED = false;
+    private const int PACKET_DUMP_START = 2825800;
+    private const int PACKET_DUMP_END = 2825900;
+    private void DumpPacketIfNecessary(Span<byte> udpPayload)
+    {
+	    if (!PACKET_DUMPING_ENABLED)
+		    return;
+
+	    if (packetSerial > PACKET_DUMP_START)
+	    {
+		    if (packetSerial < PACKET_DUMP_END)
+		    {
+			    string fname = string.Format("wne_dump/wne_{0:D4}.bin", packetSerial);
+			    FileInfo fi = new FileInfo(fname);
+			    fi.Directory.EnsureExists();
+			    File.WriteAllBytes(fname, udpPayload.ToArray());
+		    }
+	    }
+    }
     
     public void HandlePacket(InternetHeader internetHeader, byte[] ipv4Packet)
     {
 	    Span<byte> udpPayload = new Span<byte>(ipv4Packet,8,ipv4Packet.Length-8);
-
-        byte byte0 = udpPayload[0];
-        if (byte0 != 0x00)
-            return;
-        
-        byte msgFamily = udpPayload[1];
-        ushort length = udpPayload.ReadUInt16LittleEndian(2);
-        uint sessionId = udpPayload.ReadUInt32LittleEndian(4);
-        
-        if (length != udpPayload.Length && !(length == 16 && udpPayload.Length == 18))
-        {
-            return;
-        }
-
-        switch (msgFamily)
-        {
-            case 0x01:
-	            ParsePacketType1(udpPayload);
-	            break;
-            default:
-	            OnError("Unknown packet type {0:X2}", msgFamily);
-	            return;
-        }
-
-        /*if (packetSerial < 1100)
-        {
-            
-            if (outputDirectory == null)
-            {
-                outputDirectory = new DirectoryInfo("wne_dump");
-                outputDirectory.EnsureExists();
-            }
-            string fname = string.Format("wne_dump/wne_{0:D4}.bin", packetSerial);
-            File.WriteAllBytes(fname, udpPayload.ToArray());
 	    packetSerial++;
-        }*/
+	    DumpPacketIfNecessary(udpPayload);
+	    HandlePacket(udpPayload);
     }
 
-    private bool ParsePacketType1(Span<byte> udpPayload)
+    private bool HandlePacket(Span<byte> udpPayload)
     {
+	    if (packetSerial == 2825816)
+	    {
+		    
+	    }
+	    
 	    byte byte0 = udpPayload[0];
 	    if (byte0 != 0x00)
 		    return false;
@@ -96,20 +88,219 @@ public class ReutersWneExtractor : ISkyscraperMpePlugin
 	    byte msgFamily = udpPayload[1];
 	    ushort length = udpPayload.ReadUInt16LittleEndian(2);
 	    uint sessionId = udpPayload.ReadUInt32LittleEndian(4);
+        
+	    if (length != udpPayload.Length && !(length == 16 && udpPayload.Length == 18) && msgFamily != 0xff && msgFamily != 0xfe)
+	    {
+		    return false;
+	    }
+	    
+	    switch (msgFamily)
+	    {
+		    case 0x01:
+			    return ParsePacketType1(udpPayload);
+		    case 0x03:
+			    return ParsePacketType3(udpPayload);
+		    case 0xfe:
+			    return false;
+		    case 0xff:
+			    return ParsePacketType255(udpPayload);
+		    default:
+			    OnError("Unknown packet type {0:X2}", msgFamily);
+			    return false;
+	    }
+    }
+
+    private bool ParsePacketType255(Span<byte> udpPayload)
+    {
+	    byte byte0 = udpPayload[0];
+	    if (byte0 != 0x00)
+		    return false;
+	    
+	    byte msgFamily = udpPayload[1];
+	    if (msgFamily != 0xff)
+		    return false;
+	    
+	    ushort length = udpPayload.ReadUInt16LittleEndian(2);
+	    if (udpPayload.Length != length)
+	    {
+		    if (!udpPayload.Slice(8).IsBlank())
+		    {
+			    OnError("Unexpected packet length in a 0xff packet, got {0}, expected {2}.", udpPayload.Length, 8);
+			    return false;
+		    }
+	    }
+	    
+	    uint sessionId = udpPayload.ReadUInt32LittleEndian(4);
+	    
+	    WneStory currentStory = null;
+	    if (!wneStories.ContainsKey(sessionId))
+	    {
+		    OnError("Missed WNE story announcement #{0}", sessionId);
+		    return false;
+	    }
+	    else
+	    {
+		    currentStory = wneStories[sessionId];
+	    }
+
+	    if (currentStory.Delivered)
+	    {
+		    return true;
+	    }
+	    
+	    if (currentStory.Corrupted)
+	    {
+		    OnError("Story #{0} is corrupted and can not be recovered.", sessionId);
+		    wneStories.Remove(sessionId);
+		    return false;
+	    }
+
+	    if (currentStory.IsEmpty())
+	    {
+		    OnError("Story #{0} is empty and does not need to be recovered.", sessionId);
+		    wneStories.Remove(sessionId);
+		    return true;
+	    }
+
+	    DeliverFile(currentStory);
+	    currentStory.Dispose();
+	    return true;
+    }
+
+    private void DeliverFile(WneStory story)
+    {
+	    //_logger.InfoFormat("Attempting to deliver story #{0}", story.SessionId);
+	    //_logger.InfoFormat("Expected number of blocks: {0}", story.ExpectedPayloadBlock);
+
+	    ListByteArrayStream listByteArrayStream = story.ToStream();
+	    listByteArrayStream.SetLength(story.FileSize);
+	    //_logger.InfoFormat("Expected Stream length: {0} ({0:X8})", listByteArrayStream.Length);
+
+	    if (outputDirectory == null)
+	    {
+		    outputDirectory = new DirectoryInfo("wne_delivery");
+		    outputDirectory.EnsureExists();
+	    }
+
+	    string destinationFileName = story.DestinationFileName;
+	    if (destinationFileName.StartsWith("\\"))
+		    destinationFileName = destinationFileName.Substring(1);
+
+	    string outfileName = Path.Combine(outputDirectory.FullName, destinationFileName);
+	    FileInfo outFileInfo = new FileInfo(outfileName);
+		outFileInfo.Directory.EnsureExists();
+
+		FileStream fileStream = outFileInfo.OpenWrite();
+	    listByteArrayStream.CopyTo(fileStream);
+	    fileStream.Flush();
+	    fileStream.Close();
+    }
+
+    private bool ParsePacketType3(Span<byte> udpPayload)
+    {
+	    byte byte0 = udpPayload[0];
+	    if (byte0 != 0x00)
+		    return false;
+	    
+	    byte msgFamily = udpPayload[1];
+	    if (msgFamily != 0x03)
+		    return false;
+	    ushort length = udpPayload.ReadUInt16LittleEndian(2);
+	    uint sessionId = udpPayload.ReadUInt32LittleEndian(4);
+	    uint thirdUint = udpPayload.ReadUInt32LittleEndian(8);
+	    uint fourthUint = udpPayload.ReadUInt32LittleEndian(12);
+
+	    if (thirdUint != udpPayload.Length - 8)
+	    {
+		    return false;
+	    }
+
+	    if (fourthUint != udpPayload.Length - 24)
+	    {
+		    return false;
+	    }
+	    
+	    WneStory currentStory = null;
+	    if (!wneStories.ContainsKey(sessionId))
+	    {
+		    currentStory = new WneStory(sessionId);
+		    wneStories.Add(sessionId, currentStory);
+		    _logger.InfoFormat("Found new WNE story #{0}", sessionId);
+	    }
+	    else
+	    {
+		    currentStory = wneStories[sessionId];
+	    }
+
+	    byte typeDiscriminator = udpPayload[40];
+	    switch (typeDiscriminator)
+	    {
+		    case 0x02:
+			    ushort type2LengthCheck = udpPayload.ReadUInt16LittleEndian(44);
+			    if (type2LengthCheck != udpPayload.Length - 44)
+			    {
+				    OnError("Malformed packet type discriminator 2.");
+				    return false;
+			    }
+
+			    bool filenameAlreadyKnown = !string.IsNullOrEmpty(currentStory.DestinationFileName);
+			    
+			    int nextOffset = -1;
+			    currentStory.SourceFileName = udpPayload.ReadNullTerminatedUtf8String(84, out nextOffset);
+
+			    nextOffset += 55;
+			    currentStory.FileSize = udpPayload.ReadUInt32LittleEndian(nextOffset);
+			    nextOffset += 4;
+			    nextOffset += 12;
+			    
+			    currentStory.DestinationFileName = udpPayload.ReadNullTerminatedUtf8String(nextOffset, out nextOffset);
+
+			    if (!filenameAlreadyKnown)
+			    {
+				    _logger.InfoFormat("Got file announcement for story #{0}: {1}", sessionId, currentStory.DestinationFileName);
+			    }
+
+			    return true;
+		    
+		    case 0x09:
+			    currentStory.FileDeliveryType = 0x09;
+			    return true;
+		    default:
+			    OnError("Unknown packet type discriminator in a 0x03 packet {0:X2}", typeDiscriminator);
+			    return false;
+	    }
+    }
+
+    private bool ParsePacketType1(Span<byte> udpPayload)
+    {
+	    
+		byte byte0 = udpPayload[0];
+		if (byte0 != 0x00)
+			return false;
+		
+		byte msgFamily = udpPayload[1];
+		if (msgFamily != 0x01)
+			return false;
+		ushort length = udpPayload.ReadUInt16LittleEndian(2);
+		uint sessionId = udpPayload.ReadUInt32LittleEndian(4);
 		uint fourthUint = udpPayload.ReadUInt32LittleEndian(12);
 
 		if (fourthUint == 0)
 		{
-			if (length == 16)
+			if (length == 16 && udpPayload[11] == 0x07)
 			{
                 //Empty packet, likely to be used to announce stories.
 				if (!wneStories.ContainsKey(sessionId))
 				{
 					_logger.InfoFormat("Found new WNE story #{0}", sessionId);
-					WneStory newStory = new WneStory();
+					WneStory newStory = new WneStory(sessionId);
 					newStory.EccGroup = udpPayload[8];
 					if (udpPayload[11] == 0x07)
+					{
 						newStory.ExpectedEccGroup = (byte)(newStory.EccGroup + 1);
+						newStory.ExpectedContinuityCounter = 0;
+					}
+
 					wneStories.Add(sessionId, newStory);
 					return true;
 				}
@@ -122,6 +313,7 @@ public class ReutersWneExtractor : ISkyscraperMpePlugin
 					if (udpPayload[11] == 0x07)
 					{
 						currentStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1);
+						currentStory.ExpectedContinuityCounter = 0;
 					}
 					return true;
 				}
@@ -129,15 +321,174 @@ public class ReutersWneExtractor : ISkyscraperMpePlugin
 			else
 			{
 				//A/V Payload
-				OnError("A/V Payloads not supported yet.");
+				if (!wneStories.ContainsKey(sessionId))
+				{
+					OnError("Missed the announcement for story #{0}. Data blocks of it are therefore not usable.", sessionId);
 					return false;
 				}
+
+				if (fourthUint != 0)
+				{
+					OnError("Unexpected payload packet in story #{0}. Expected {1}, got {2}.", sessionId, 0, fourthUint);
+					return false;
+				}
+				uint thirdUint = udpPayload.ReadUInt32LittleEndian(8);
+				if (thirdUint != wneStories[sessionId].ExpectedPayloadBlock)
+				{
+					if (!wneStories[sessionId].Corrupted)
+					{
+						OnError("Expected payload block {0} but got {1}. This story ({2}) is incomplete and can not be recovered.", wneStories[sessionId].ExpectedPayloadBlock, thirdUint, sessionId);
+						wneStories[sessionId].Corrupted = true;
+					}
+					return false;
+				}
+				
+				wneStories[sessionId].AppendPayloadBlock(udpPayload.Slice(16));
+				return true;
+			}
 		}
 		else
 		{
-			OnError("Non-A/V Payloads not supported yet.");
+			WneStory outerStory = null;
+			if (!wneStories.ContainsKey(sessionId))
+			{
+				OnError("Missed announcement of story #{0}", sessionId);
 				return false;
 			}
+			outerStory = wneStories[sessionId];
+			
+			bool payloadIsEcc = false;
+			if (udpPayload[12] == 0x07)
+			{
+				outerStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1);
+				outerStory.ExpectedContinuityCounter = 0;
+				payloadIsEcc = true;
+			}
+
+			if (payloadIsEcc)
+			{
+				//OnError("ECC Payloads not supported yet.");
+				return false;
+			}
+
+			int lengthCheckTolerance = 15;
+			bool hasAdditionalHeader = false;
+			if (udpPayload[12] == 0x04)
+			{
+				lengthCheckTolerance = 8;
+				hasAdditionalHeader = true;
+			}
+			
+			
+			ushort embeddedPacketLengthCheck = udpPayload.ReadUInt16LittleEndian(14);
+			if (embeddedPacketLengthCheck == length - lengthCheckTolerance)
+			{
+				
+				uint embeddedSessionId = udpPayload.ReadUInt32LittleEndian(20);
+				if (embeddedSessionId == 0)
+					embeddedSessionId = sessionId;
+				if (!wneStories.ContainsKey(embeddedSessionId))
+				{
+					byte embeddedPacketType = udpPayload[17];
+					if (embeddedPacketType == 0x03)
+					{
+						wneStories.Add(embeddedSessionId, new WneStory(embeddedSessionId));
+						_logger.InfoFormat("Found new embedded WNE story #{0}", embeddedSessionId);
+					}
+					else
+					{
+						OnError("Missed announcement of embedded WNE story #{0}", embeddedSessionId);
+						return false;
+					}
+				}
+
+				if (outerStory.timesSucessfullySynced == 0)
+				{
+					outerStory.ExpectedEccGroup = udpPayload[8];
+					outerStory.ExpectedContinuityCounter = udpPayload[11] + 1;
+					outerStory.timesSucessfullySynced++;
+				}
+				else
+				{
+					if (outerStory.ExpectedEccGroup != udpPayload[8])
+					{
+						OnError("Expected ECC group {0} but got {1}", outerStory.ExpectedEccGroup, udpPayload[8]);
+						return false;
+					}
+
+					if (udpPayload[0x000b] == 0x07 && udpPayload[0x000c] == 0x03)
+					{
+						outerStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1);
+						outerStory.ExpectedContinuityCounter = 0;
+						return true;
+					}
+
+					bool isFileFinished = udpPayload[18] == 0xff;
+
+					if (udpPayload[11] != 0x07 || udpPayload[12] != 0x04)
+					{
+						if (isFileFinished)
+						{
+							if (udpPayload[12] != 0x01)
+							{
+								OnError("Expected Continuity Counter {0} but got {1}",
+									outerStory.ExpectedContinuityCounter, udpPayload[11]);
+								return false;
+							}
+
+							outerStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1);
+							outerStory.ExpectedContinuityCounter = 0;
+							return true;
+						}
+						else
+						{
+							outerStory.ExpectedEccGroup = udpPayload[8];
+							outerStory.ExpectedContinuityCounter = udpPayload[11] + 1;
+						}
+					}
+					else
+					{
+						outerStory.ExpectedEccGroup = udpPayload[8];
+						outerStory.ExpectedContinuityCounter = udpPayload[11] + 1;
+						outerStory.timesSucessfullySynced++;
+					}
+				}
+
+				if (udpPayload[11] == 0x07)
+				{
+					outerStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1);
+					outerStory.ExpectedContinuityCounter = 0;
+				}
+				
+				Span<byte> embeddedPayload = udpPayload.Slice(16);
+				return HandlePacket(embeddedPayload);
+					
+			}
+			else
+			{
+				if (udpPayload.Slice(16).IsBlank())
+				{
+					return true;
+				}
+
+				if (udpPayload[12] == 0x05 && udpPayload[14] == 0x09)
+				{
+					uint thirdUint = udpPayload.ReadUInt32LittleEndian(8);
+					if (thirdUint != wneStories[sessionId].ExpectedPayloadBlock)
+					{
+						OnError("Expected payload block {0} but got {1}. This story ({2}) is incomplete and can not be recovered.", wneStories[sessionId].ExpectedPayloadBlock, thirdUint, sessionId);
+						wneStories[sessionId].Corrupted = true;
+						return false;
+					}
+				
+					wneStories[sessionId].AppendPayloadBlock(udpPayload.Slice(16));
+					return true;
+				}
+				
+				OnError("Unknown packet type in a 0x01 packet.");
+				return false;
+			}
+		}
     }
 
     public bool StopProcessingAfterThis()
@@ -154,6 +505,7 @@ public class ReutersWneExtractor : ISkyscraperMpePlugin
 		    loggedErrors = new HashSet<string>();
 
 	    string unpacked = String.Format(message, args);
+	    unpacked = String.Format("Packet #{0}: {1}", packetSerial, unpacked);
 	    if (!loggedErrors.Contains(unpacked))
 	    {
 		    loggedErrors.Add(unpacked);
@@ -161,10 +513,5 @@ public class ReutersWneExtractor : ISkyscraperMpePlugin
 	    }
     }
 
-    private class WneStory
+    
-    {
-	    public byte EccGroup;
-	    public byte ExpectedEccGroup;
-	    public int timesSucessfullySynced;
-    }
 }

skyscraper8/ReutersWne/WneStory.cs

@@ -0,0 +1,74 @@
+namespace skyscraper8.ReutersWne;
+
+internal class WneStory : IDisposable
+{
+    public uint SessionId { get; }
+
+    public WneStory(uint sessionId)
+    {
+        SessionId = sessionId;
+    }
+    
+    public byte EccGroup;
+    public byte ExpectedEccGroup;
+    public int timesSucessfullySynced;
+    public int FileDeliveryType;
+    public int ExpectedContinuityCounter;
+    public string SourceFileName;
+    public string DestinationFileName;
+    public uint ExpectedPayloadBlock;
+
+    private List<byte[]> payloadBlocks;
+    public bool Corrupted;
+    public bool Delivered;
+    public uint FileSize;
+
+    public void AppendPayloadBlock(Span<byte> slice)
+    {
+        if (payloadBlocks == null)
+        {
+            payloadBlocks = new List<byte[]>();
+        }
+        payloadBlocks.Add(slice.ToArray());
+        ExpectedPayloadBlock++;
+    }
+
+    public bool IsEmpty()
+    {
+        if (payloadBlocks == null)
+        {
+            return true;
+        }
+
+        if (payloadBlocks.Count == 0)
+        {
+            return true;
+        }
+        
+        for (int i = 0; i < payloadBlocks.Count; i++)
+        {
+            if (payloadBlocks[i].Length > 0)
+            {
+                return false;
+            }
+        }
+        
+        return true;
+    }
+
+    public ListByteArrayStream ToStream()
+    {
+        if (Delivered)
+        {
+            throw new InvalidOperationException("Cannot convert a delivered story to a stream.");
+        }
+        return new ListByteArrayStream(payloadBlocks);
+    }
+
+    public void Dispose()
+    {
+        payloadBlocks?.Clear();
+        payloadBlocks = null;
+        Delivered = true;
+    }
+}

skyscraper8/Skyscraper/SpanByteExtensions.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Text;
 
 public static class SpanByteExtensions
 {
@@ -58,5 +59,49 @@ public static class SpanByteExtensions
     public static uint ReadUInt32BigEndian(this Span<byte> span, int offset)
         => ReadUInt32BigEndian((ReadOnlySpan<byte>)span, offset);
     
+    public static string ReadNullTerminatedUtf8String(this ReadOnlySpan<byte> span, int offset)
+    {
+        if ((uint)offset >= (uint)span.Length)
+            throw new ArgumentOutOfRangeException(nameof(offset));
 
+        int nullIndex = span.Slice(offset).IndexOf((byte)0);
+        if (nullIndex < 0)
+            throw new InvalidOperationException("No null terminator found in span.");
+
+        return Encoding.UTF8.GetString(span.Slice(offset, nullIndex));
+    }
+
+    public static string ReadNullTerminatedUtf8String(this ReadOnlySpan<byte> span, int offset, out int nextOffset)
+    {
+        if ((uint)offset >= (uint)span.Length)
+            throw new ArgumentOutOfRangeException(nameof(offset));
+
+        int nullIndex = span.Slice(offset).IndexOf((byte)0);
+        if (nullIndex < 0)
+            throw new InvalidOperationException("No null terminator found in span.");
+
+        nextOffset = offset + nullIndex + 1;
+        return Encoding.UTF8.GetString(span.Slice(offset, nullIndex));
+    }
+
+    public static string ReadNullTerminatedUtf8String(this Span<byte> span, int offset)
+        => ReadNullTerminatedUtf8String((ReadOnlySpan<byte>)span, offset);
+
+    public static string ReadNullTerminatedUtf8String(this Span<byte> span, int offset, out int nextOffset)
+        => ReadNullTerminatedUtf8String((ReadOnlySpan<byte>)span, offset, out nextOffset);
+
+    public static bool IsBlank(this Span<byte> span)
+    {
+        if (span.Length == 0)
+            return true;
+        else
+        {
+            for (int i = 0; i < span.Length; i++)
+            {
+                if (span[i] != 0)
+                    return false;
+            }
+            return true;
+        }
+    }
 }