From e79ef4b4e229ebf3003724aee06c3453649b87a2 Mon Sep 17 00:00:00 2001 From: feyris-tan <4116042+feyris-tan@users.noreply.github.com> Date: Sat, 9 May 2026 15:23:20 +0200 Subject: [PATCH] Newlines in ReutersWneExtractor --- skyscraper8/ReutersWne/ReutersWneExtractor.cs | 1072 ++++++++--------- 1 file changed, 536 insertions(+), 536 deletions(-) diff --git a/skyscraper8/ReutersWne/ReutersWneExtractor.cs b/skyscraper8/ReutersWne/ReutersWneExtractor.cs index 038578c..68e9fce 100644 --- a/skyscraper8/ReutersWne/ReutersWneExtractor.cs +++ b/skyscraper8/ReutersWne/ReutersWneExtractor.cs @@ -1,544 +1,544 @@ -using log4net; -using skyscraper5.Ietf.Rfc971; -using skyscraper5.Skyscraper.IO; -using skyscraper5.Skyscraper.Plugins; - -namespace skyscraper8.ReutersWne; - -[SkyscraperPlugin] -internal class ReutersWneExtractor : ISkyscraperMpePlugin -{ - private ILog _logger; - public void ConnectToStorage(object[] connector) - { - _logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); - //TODO: connect to stroage plugins - } - - public void SetContext(DateTime? currentTime, object skyscraperContext) - { - //TODO: remember current time and skyscraper context - if (wneStories == null) - wneStories = new Dictionary(); - - IWneHandler contextableWneHandler = skyscraperContext as IWneHandler; +using log4net; +using skyscraper5.Ietf.Rfc971; +using skyscraper5.Skyscraper.IO; +using skyscraper5.Skyscraper.Plugins; + +namespace skyscraper8.ReutersWne; + +[SkyscraperPlugin] +internal class ReutersWneExtractor : ISkyscraperMpePlugin +{ + private ILog _logger; + public void ConnectToStorage(object[] connector) + { + _logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); + //TODO: connect to stroage plugins + } + + public void SetContext(DateTime? currentTime, object skyscraperContext) + { + //TODO: remember current time and skyscraper context + if (wneStories == null) + wneStories = new Dictionary(); + + IWneHandler contextableWneHandler = skyscraperContext as IWneHandler; if (contextableWneHandler != null) { this.Handler = contextableWneHandler; - } + } else { _logger.ErrorFormat("The provided SkyscraperContext does not support handling WNE stories."); - } - } - - public bool CanHandlePacket(InternetHeader internetHeader, byte[] ipv4Packet) - { - if (internetHeader.Protocol != 0x11) //Reuters WNE is UDP only - return false; - if (ipv4Packet[8] != 0) //First opcode byte is always "0" - return false; - - //These bytes are according to the "WNE - RX8200 Configuration.pdf", page 11 - byte[] addressBytes = internetHeader.DestinationAddress.GetAddressBytes(); - if (addressBytes[0] != 224) - return false; - if (addressBytes[1] != 14) - return false; - if (addressBytes[2] != 14) - return false; - if (addressBytes[3] > 16) //Should be 14 or 15 - return false; - - return true; - } - - private DirectoryInfo outputDirectory; - private ulong packetSerial = 0; - - private const bool PACKET_DUMPING_ENABLED = false; - private const int PACKET_DUMP_START = 2825800; - private const int PACKET_DUMP_END = 2825900; - private void DumpPacketIfNecessary(Span udpPayload) - { - if (!PACKET_DUMPING_ENABLED) - return; - - if (packetSerial > PACKET_DUMP_START) - { - if (packetSerial < PACKET_DUMP_END) - { - string fname = string.Format("wne_dump/wne_{0:D4}.bin", packetSerial); - FileInfo fi = new FileInfo(fname); - fi.Directory.EnsureExists(); - File.WriteAllBytes(fname, udpPayload.ToArray()); - } - } - } - - public void HandlePacket(InternetHeader internetHeader, byte[] ipv4Packet) - { - Span udpPayload = new Span(ipv4Packet,8,ipv4Packet.Length-8); - packetSerial++; - DumpPacketIfNecessary(udpPayload); - HandlePacket(udpPayload); - } - - private bool HandlePacket(Span udpPayload) - { - if (packetSerial == 2825816) - { - - } - - byte byte0 = udpPayload[0]; - if (byte0 != 0x00) - return false; - - byte msgFamily = udpPayload[1]; - ushort length = udpPayload.ReadUInt16LittleEndian(2); - uint sessionId = udpPayload.ReadUInt32LittleEndian(4); - - if (length != udpPayload.Length && !(length == 16 && udpPayload.Length == 18) && msgFamily != 0xff && msgFamily != 0xfe) - { - return false; - } - - switch (msgFamily) - { - case 0x01: - return ParsePacketType1(udpPayload); - case 0x03: - return ParsePacketType3(udpPayload); - case 0xfe: - return false; - case 0xff: - return ParsePacketType255(udpPayload); - default: - OnError("Unknown packet type {0:X2}", msgFamily); - return false; - } - } - - private bool ParsePacketType255(Span udpPayload) - { - byte byte0 = udpPayload[0]; - if (byte0 != 0x00) - return false; - - byte msgFamily = udpPayload[1]; - if (msgFamily != 0xff) - return false; - - ushort length = udpPayload.ReadUInt16LittleEndian(2); - if (udpPayload.Length != length) - { - if (!udpPayload.Slice(8).IsBlank()) - { - OnError("Unexpected packet length in a 0xff packet, got {0}, expected {2}.", udpPayload.Length, 8); - return false; - } - } - - uint sessionId = udpPayload.ReadUInt32LittleEndian(4); - - WneStory currentStory = null; - if (!wneStories.ContainsKey(sessionId)) - { - OnError("Missed WNE story announcement #{0}", sessionId); - return false; - } - else - { - currentStory = wneStories[sessionId]; - } - - if (currentStory.Delivered) - { - return true; - } - - if (currentStory.Corrupted) - { - OnError("Story #{0} is corrupted and can not be recovered.", sessionId); - currentStory.Dispose(); - wneStories.Remove(sessionId); - return false; - } - - if (currentStory.IsEmpty()) - { - OnError("Story #{0} is empty and does not need to be recovered.", sessionId); - wneStories.Remove(sessionId); - return true; - } - - DeliverFile(currentStory); - currentStory.Dispose(); - return true; - } - - private void DeliverFile(WneStory story) - { - Handler?.OnWneStoryComplete(new WneStoryProgress(story), story.ToStream()); - _logger.InfoFormat("Extracted file: {0}", story.DestinationFileName); - - /* - //_logger.InfoFormat("Attempting to deliver story #{0}", story.SessionId); - //_logger.InfoFormat("Expected number of blocks: {0}", story.ExpectedPayloadBlock); - - ListByteArrayStream listByteArrayStream = story.ToStream(); - listByteArrayStream.SetLength(story.FileSize); - //_logger.InfoFormat("Expected Stream length: {0} ({0:X8})", listByteArrayStream.Length); - - if (outputDirectory == null) - { - outputDirectory = new DirectoryInfo("wne_delivery"); - outputDirectory.EnsureExists(); - } - - string destinationFileName = story.DestinationFileName; - if (destinationFileName.StartsWith("\\")) - destinationFileName = destinationFileName.Substring(1); - - string outfileName = Path.Combine(outputDirectory.FullName, destinationFileName); - FileInfo outFileInfo = new FileInfo(outfileName); - outFileInfo.Directory.EnsureExists(); - - FileStream fileStream = outFileInfo.OpenWrite(); - listByteArrayStream.CopyTo(fileStream); - fileStream.Flush(); - fileStream.Close();*/ - } - - private bool ParsePacketType3(Span udpPayload) - { - byte byte0 = udpPayload[0]; - if (byte0 != 0x00) - return false; - - byte msgFamily = udpPayload[1]; - if (msgFamily != 0x03) - return false; - ushort length = udpPayload.ReadUInt16LittleEndian(2); - uint sessionId = udpPayload.ReadUInt32LittleEndian(4); - uint thirdUint = udpPayload.ReadUInt32LittleEndian(8); - uint fourthUint = udpPayload.ReadUInt32LittleEndian(12); - - if (thirdUint != udpPayload.Length - 8) - { - return false; - } - - if (fourthUint != udpPayload.Length - 24) - { - return false; - } - - WneStory currentStory = null; - if (!wneStories.ContainsKey(sessionId)) - { - currentStory = new WneStory(sessionId); - wneStories.Add(sessionId, currentStory); - _logger.InfoFormat("Found new WNE story #{0}", sessionId); - Handler?.OnWneStoryDetect(sessionId); - } - else - { - currentStory = wneStories[sessionId]; - } - - byte typeDiscriminator = udpPayload[40]; - switch (typeDiscriminator) - { - case 0x02: - ushort type2LengthCheck = udpPayload.ReadUInt16LittleEndian(44); - if (type2LengthCheck != udpPayload.Length - 44) - { - OnError("Malformed packet type discriminator 2."); - return false; - } - - bool filenameAlreadyKnown = !string.IsNullOrEmpty(currentStory.DestinationFileName); - - int nextOffset = -1; - currentStory.SourceFileName = udpPayload.ReadNullTerminatedUtf8String(84, out nextOffset); - - nextOffset += 55; - currentStory.FileSize = udpPayload.ReadUInt32LittleEndian(nextOffset); - nextOffset += 4; - nextOffset += 12; - - currentStory.DestinationFileName = udpPayload.ReadNullTerminatedUtf8String(nextOffset, out nextOffset); - - if (!filenameAlreadyKnown) - { - _logger.InfoFormat("Got file announcement for story #{0}: {1}", sessionId, currentStory.DestinationFileName); - } - - return true; - - case 0x09: - currentStory.FileDeliveryType = 0x09; - return true; - default: - OnError("Unknown packet type discriminator in a 0x03 packet {0:X2}", typeDiscriminator); - return false; - } - } - - private bool ParsePacketType1(Span udpPayload) - { - - byte byte0 = udpPayload[0]; - if (byte0 != 0x00) - return false; - - byte msgFamily = udpPayload[1]; - if (msgFamily != 0x01) - return false; - ushort length = udpPayload.ReadUInt16LittleEndian(2); - uint sessionId = udpPayload.ReadUInt32LittleEndian(4); - uint fourthUint = udpPayload.ReadUInt32LittleEndian(12); - - if (fourthUint == 0) - { - if (length == 16 && udpPayload[11] == 0x07) - { - //Empty packet, likely to be used to announce stories. - if (!wneStories.ContainsKey(sessionId)) - { - _logger.InfoFormat("Found new WNE story #{0}", sessionId); - Handler?.OnWneStoryDetect(sessionId); - WneStory newStory = new WneStory(sessionId); - newStory.EccGroup = udpPayload[8]; - if (udpPayload[11] == 0x07) - { - newStory.ExpectedEccGroup = (byte)(newStory.EccGroup + 1); - newStory.ExpectedContinuityCounter = 0; - } - - wneStories.Add(sessionId, newStory); - return true; - } - else - { - WneStory currentStory = wneStories[sessionId]; - if (udpPayload[8] == currentStory.ExpectedEccGroup) - currentStory.timesSucessfullySynced++; - currentStory.EccGroup = udpPayload[8]; - if (udpPayload[11] == 0x07) - { - currentStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1); - currentStory.ExpectedContinuityCounter = 0; - } - return true; - } - } - else - { - //A/V Payload - if (!wneStories.ContainsKey(sessionId)) - { - OnError("Missed the announcement for story #{0}. Data blocks of it are therefore not usable.", sessionId); - return false; - } - - if (fourthUint != 0) - { - OnError("Unexpected payload packet in story #{0}. Expected {1}, got {2}.", sessionId, 0, fourthUint); - Handler?.OnWneStoryFail(sessionId); - return false; - } - uint thirdUint = udpPayload.ReadUInt32LittleEndian(8); - if (thirdUint != wneStories[sessionId].ExpectedPayloadBlock) - { - if (!wneStories[sessionId].Corrupted) - { - OnError("Expected payload block {0} but got {1}. This story ({2}) is incomplete and can not be recovered.", wneStories[sessionId].ExpectedPayloadBlock, thirdUint, sessionId); - wneStories[sessionId].Corrupted = true; - Handler?.OnWneStoryFail(sessionId); - } - return false; - } - - wneStories[sessionId].AppendPayloadBlock(udpPayload.Slice(16)); - Handler?.OnWneStoryProgress(new WneStoryProgress(wneStories[sessionId])); - return true; - } - } - else - { - WneStory outerStory = null; - if (!wneStories.ContainsKey(sessionId)) - { - OnError("Missed announcement of story #{0}", sessionId); - Handler.OnWneStoryFail(sessionId); - return false; - } - outerStory = wneStories[sessionId]; - - bool payloadIsEcc = false; - if (udpPayload[12] == 0x07) - { - outerStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1); - outerStory.ExpectedContinuityCounter = 0; - payloadIsEcc = true; - } - - if (payloadIsEcc) - { - //OnError("ECC Payloads not supported yet."); - return false; - } - - int lengthCheckTolerance = 15; - bool hasAdditionalHeader = false; - if (udpPayload[12] == 0x04) - { - lengthCheckTolerance = 8; - hasAdditionalHeader = true; - } - - - ushort embeddedPacketLengthCheck = udpPayload.ReadUInt16LittleEndian(14); - if (embeddedPacketLengthCheck == length - lengthCheckTolerance) - { - - uint embeddedSessionId = udpPayload.ReadUInt32LittleEndian(20); - if (embeddedSessionId == 0) - embeddedSessionId = sessionId; - if (!wneStories.ContainsKey(embeddedSessionId)) - { - byte embeddedPacketType = udpPayload[17]; - if (embeddedPacketType == 0x03) - { - wneStories.Add(embeddedSessionId, new WneStory(embeddedSessionId)); - _logger.InfoFormat("Found new embedded WNE story #{0}", embeddedSessionId); - Handler?.OnWneStoryDetect(embeddedSessionId); - } - else - { - OnError("Missed announcement of embedded WNE story #{0}", embeddedSessionId); - Handler?.OnWneStoryFail(outerStory.SessionId); - return false; - } - } - - if (outerStory.timesSucessfullySynced == 0) - { - outerStory.ExpectedEccGroup = udpPayload[8]; - outerStory.ExpectedContinuityCounter = udpPayload[11] + 1; - outerStory.timesSucessfullySynced++; - } - else - { - if (outerStory.ExpectedEccGroup != udpPayload[8]) - { - OnError("Expected ECC group {0} but got {1}", outerStory.ExpectedEccGroup, udpPayload[8]); - Handler?.OnWneStoryFail(outerStory.SessionId); - return false; - } - - if (udpPayload[0x000b] == 0x07 && udpPayload[0x000c] == 0x03) - { - outerStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1); - outerStory.ExpectedContinuityCounter = 0; - return true; - } - - bool isFileFinished = udpPayload[18] == 0xff; - - if (udpPayload[11] != 0x07 || udpPayload[12] != 0x04) - { - if (isFileFinished) - { - if (udpPayload[12] != 0x01) - { - OnError("Expected Continuity Counter {0} but got {1}", outerStory.ExpectedContinuityCounter, udpPayload[11]); - Handler?.OnWneStoryFail(outerStory.SessionId); - return false; - } - - outerStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1); - outerStory.ExpectedContinuityCounter = 0; - return true; - } - else - { - outerStory.ExpectedEccGroup = udpPayload[8]; - outerStory.ExpectedContinuityCounter = udpPayload[11] + 1; - } - } - else - { - outerStory.ExpectedEccGroup = udpPayload[8]; - outerStory.ExpectedContinuityCounter = udpPayload[11] + 1; - outerStory.timesSucessfullySynced++; - } - } - - if (udpPayload[11] == 0x07) - { - outerStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1); - outerStory.ExpectedContinuityCounter = 0; - } - - Span embeddedPayload = udpPayload.Slice(16); - return HandlePacket(embeddedPayload); - - } - else - { - if (udpPayload.Slice(16).IsBlank()) - { - return true; - } - - if (udpPayload[12] == 0x05 && udpPayload[14] == 0x09) - { - uint thirdUint = udpPayload.ReadUInt32LittleEndian(8); - if (thirdUint != wneStories[sessionId].ExpectedPayloadBlock) - { - OnError("Expected payload block {0} but got {1}. This story ({2}) is incomplete and can not be recovered.", wneStories[sessionId].ExpectedPayloadBlock, thirdUint, sessionId); - wneStories[sessionId].Corrupted = true; - Handler?.OnWneStoryFail(sessionId); - return false; - } - - wneStories[sessionId].AppendPayloadBlock(udpPayload.Slice(16)); - Handler?.OnWneStoryProgress(new WneStoryProgress(wneStories[sessionId])); - return true; - } - - OnError("Unknown packet type in a 0x01 packet."); - return false; - } - } - } - - public bool StopProcessingAfterThis() - { - return true; - } - - private HashSet loggedErrors; - private Dictionary wneStories; - public IWneHandler Handler { get; set; } - - private void OnError(string message, params object[] args) - { - if (loggedErrors == null) - loggedErrors = new HashSet(); - - string unpacked = String.Format(message, args); - if (!loggedErrors.Contains(unpacked)) - { + } + } + + public bool CanHandlePacket(InternetHeader internetHeader, byte[] ipv4Packet) + { + if (internetHeader.Protocol != 0x11) //Reuters WNE is UDP only + return false; + if (ipv4Packet[8] != 0) //First opcode byte is always "0" + return false; + + //These bytes are according to the "WNE - RX8200 Configuration.pdf", page 11 + byte[] addressBytes = internetHeader.DestinationAddress.GetAddressBytes(); + if (addressBytes[0] != 224) + return false; + if (addressBytes[1] != 14) + return false; + if (addressBytes[2] != 14) + return false; + if (addressBytes[3] > 16) //Should be 14 or 15 + return false; + + return true; + } + + private DirectoryInfo outputDirectory; + private ulong packetSerial = 0; + + private const bool PACKET_DUMPING_ENABLED = false; + private const int PACKET_DUMP_START = 2825800; + private const int PACKET_DUMP_END = 2825900; + private void DumpPacketIfNecessary(Span udpPayload) + { + if (!PACKET_DUMPING_ENABLED) + return; + + if (packetSerial > PACKET_DUMP_START) + { + if (packetSerial < PACKET_DUMP_END) + { + string fname = string.Format("wne_dump/wne_{0:D4}.bin", packetSerial); + FileInfo fi = new FileInfo(fname); + fi.Directory.EnsureExists(); + File.WriteAllBytes(fname, udpPayload.ToArray()); + } + } + } + + public void HandlePacket(InternetHeader internetHeader, byte[] ipv4Packet) + { + Span udpPayload = new Span(ipv4Packet,8,ipv4Packet.Length-8); + packetSerial++; + DumpPacketIfNecessary(udpPayload); + HandlePacket(udpPayload); + } + + private bool HandlePacket(Span udpPayload) + { + if (packetSerial == 2825816) + { + + } + + byte byte0 = udpPayload[0]; + if (byte0 != 0x00) + return false; + + byte msgFamily = udpPayload[1]; + ushort length = udpPayload.ReadUInt16LittleEndian(2); + uint sessionId = udpPayload.ReadUInt32LittleEndian(4); + + if (length != udpPayload.Length && !(length == 16 && udpPayload.Length == 18) && msgFamily != 0xff && msgFamily != 0xfe) + { + return false; + } + + switch (msgFamily) + { + case 0x01: + return ParsePacketType1(udpPayload); + case 0x03: + return ParsePacketType3(udpPayload); + case 0xfe: + return false; + case 0xff: + return ParsePacketType255(udpPayload); + default: + OnError("Unknown packet type {0:X2}", msgFamily); + return false; + } + } + + private bool ParsePacketType255(Span udpPayload) + { + byte byte0 = udpPayload[0]; + if (byte0 != 0x00) + return false; + + byte msgFamily = udpPayload[1]; + if (msgFamily != 0xff) + return false; + + ushort length = udpPayload.ReadUInt16LittleEndian(2); + if (udpPayload.Length != length) + { + if (!udpPayload.Slice(8).IsBlank()) + { + OnError("Unexpected packet length in a 0xff packet, got {0}, expected {2}.", udpPayload.Length, 8); + return false; + } + } + + uint sessionId = udpPayload.ReadUInt32LittleEndian(4); + + WneStory currentStory = null; + if (!wneStories.ContainsKey(sessionId)) + { + OnError("Missed WNE story announcement #{0}", sessionId); + return false; + } + else + { + currentStory = wneStories[sessionId]; + } + + if (currentStory.Delivered) + { + return true; + } + + if (currentStory.Corrupted) + { + OnError("Story #{0} is corrupted and can not be recovered.", sessionId); + currentStory.Dispose(); + wneStories.Remove(sessionId); + return false; + } + + if (currentStory.IsEmpty()) + { + OnError("Story #{0} is empty and does not need to be recovered.", sessionId); + wneStories.Remove(sessionId); + return true; + } + + DeliverFile(currentStory); + currentStory.Dispose(); + return true; + } + + private void DeliverFile(WneStory story) + { + Handler?.OnWneStoryComplete(new WneStoryProgress(story), story.ToStream()); + _logger.InfoFormat("Extracted file: {0}", story.DestinationFileName); + + /* + //_logger.InfoFormat("Attempting to deliver story #{0}", story.SessionId); + //_logger.InfoFormat("Expected number of blocks: {0}", story.ExpectedPayloadBlock); + + ListByteArrayStream listByteArrayStream = story.ToStream(); + listByteArrayStream.SetLength(story.FileSize); + //_logger.InfoFormat("Expected Stream length: {0} ({0:X8})", listByteArrayStream.Length); + + if (outputDirectory == null) + { + outputDirectory = new DirectoryInfo("wne_delivery"); + outputDirectory.EnsureExists(); + } + + string destinationFileName = story.DestinationFileName; + if (destinationFileName.StartsWith("\\")) + destinationFileName = destinationFileName.Substring(1); + + string outfileName = Path.Combine(outputDirectory.FullName, destinationFileName); + FileInfo outFileInfo = new FileInfo(outfileName); + outFileInfo.Directory.EnsureExists(); + + FileStream fileStream = outFileInfo.OpenWrite(); + listByteArrayStream.CopyTo(fileStream); + fileStream.Flush(); + fileStream.Close();*/ + } + + private bool ParsePacketType3(Span udpPayload) + { + byte byte0 = udpPayload[0]; + if (byte0 != 0x00) + return false; + + byte msgFamily = udpPayload[1]; + if (msgFamily != 0x03) + return false; + ushort length = udpPayload.ReadUInt16LittleEndian(2); + uint sessionId = udpPayload.ReadUInt32LittleEndian(4); + uint thirdUint = udpPayload.ReadUInt32LittleEndian(8); + uint fourthUint = udpPayload.ReadUInt32LittleEndian(12); + + if (thirdUint != udpPayload.Length - 8) + { + return false; + } + + if (fourthUint != udpPayload.Length - 24) + { + return false; + } + + WneStory currentStory = null; + if (!wneStories.ContainsKey(sessionId)) + { + currentStory = new WneStory(sessionId); + wneStories.Add(sessionId, currentStory); + _logger.InfoFormat("Found new WNE story #{0}", sessionId); + Handler?.OnWneStoryDetect(sessionId); + } + else + { + currentStory = wneStories[sessionId]; + } + + byte typeDiscriminator = udpPayload[40]; + switch (typeDiscriminator) + { + case 0x02: + ushort type2LengthCheck = udpPayload.ReadUInt16LittleEndian(44); + if (type2LengthCheck != udpPayload.Length - 44) + { + OnError("Malformed packet type discriminator 2."); + return false; + } + + bool filenameAlreadyKnown = !string.IsNullOrEmpty(currentStory.DestinationFileName); + + int nextOffset = -1; + currentStory.SourceFileName = udpPayload.ReadNullTerminatedUtf8String(84, out nextOffset); + + nextOffset += 55; + currentStory.FileSize = udpPayload.ReadUInt32LittleEndian(nextOffset); + nextOffset += 4; + nextOffset += 12; + + currentStory.DestinationFileName = udpPayload.ReadNullTerminatedUtf8String(nextOffset, out nextOffset); + + if (!filenameAlreadyKnown) + { + _logger.InfoFormat("Got file announcement for story #{0}: {1}", sessionId, currentStory.DestinationFileName); + } + + return true; + + case 0x09: + currentStory.FileDeliveryType = 0x09; + return true; + default: + OnError("Unknown packet type discriminator in a 0x03 packet {0:X2}", typeDiscriminator); + return false; + } + } + + private bool ParsePacketType1(Span udpPayload) + { + + byte byte0 = udpPayload[0]; + if (byte0 != 0x00) + return false; + + byte msgFamily = udpPayload[1]; + if (msgFamily != 0x01) + return false; + ushort length = udpPayload.ReadUInt16LittleEndian(2); + uint sessionId = udpPayload.ReadUInt32LittleEndian(4); + uint fourthUint = udpPayload.ReadUInt32LittleEndian(12); + + if (fourthUint == 0) + { + if (length == 16 && udpPayload[11] == 0x07) + { + //Empty packet, likely to be used to announce stories. + if (!wneStories.ContainsKey(sessionId)) + { + _logger.InfoFormat("Found new WNE story #{0}", sessionId); + Handler?.OnWneStoryDetect(sessionId); + WneStory newStory = new WneStory(sessionId); + newStory.EccGroup = udpPayload[8]; + if (udpPayload[11] == 0x07) + { + newStory.ExpectedEccGroup = (byte)(newStory.EccGroup + 1); + newStory.ExpectedContinuityCounter = 0; + } + + wneStories.Add(sessionId, newStory); + return true; + } + else + { + WneStory currentStory = wneStories[sessionId]; + if (udpPayload[8] == currentStory.ExpectedEccGroup) + currentStory.timesSucessfullySynced++; + currentStory.EccGroup = udpPayload[8]; + if (udpPayload[11] == 0x07) + { + currentStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1); + currentStory.ExpectedContinuityCounter = 0; + } + return true; + } + } + else + { + //A/V Payload + if (!wneStories.ContainsKey(sessionId)) + { + OnError("Missed the announcement for story #{0}. Data blocks of it are therefore not usable.", sessionId); + return false; + } + + if (fourthUint != 0) + { + OnError("Unexpected payload packet in story #{0}. Expected {1}, got {2}.", sessionId, 0, fourthUint); + Handler?.OnWneStoryFail(sessionId); + return false; + } + uint thirdUint = udpPayload.ReadUInt32LittleEndian(8); + if (thirdUint != wneStories[sessionId].ExpectedPayloadBlock) + { + if (!wneStories[sessionId].Corrupted) + { + OnError("Expected payload block {0} but got {1}. This story ({2}) is incomplete and can not be recovered.", wneStories[sessionId].ExpectedPayloadBlock, thirdUint, sessionId); + wneStories[sessionId].Corrupted = true; + Handler?.OnWneStoryFail(sessionId); + } + return false; + } + + wneStories[sessionId].AppendPayloadBlock(udpPayload.Slice(16)); + Handler?.OnWneStoryProgress(new WneStoryProgress(wneStories[sessionId])); + return true; + } + } + else + { + WneStory outerStory = null; + if (!wneStories.ContainsKey(sessionId)) + { + OnError("Missed announcement of story #{0}", sessionId); + Handler.OnWneStoryFail(sessionId); + return false; + } + outerStory = wneStories[sessionId]; + + bool payloadIsEcc = false; + if (udpPayload[12] == 0x07) + { + outerStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1); + outerStory.ExpectedContinuityCounter = 0; + payloadIsEcc = true; + } + + if (payloadIsEcc) + { + //OnError("ECC Payloads not supported yet."); + return false; + } + + int lengthCheckTolerance = 15; + bool hasAdditionalHeader = false; + if (udpPayload[12] == 0x04) + { + lengthCheckTolerance = 8; + hasAdditionalHeader = true; + } + + + ushort embeddedPacketLengthCheck = udpPayload.ReadUInt16LittleEndian(14); + if (embeddedPacketLengthCheck == length - lengthCheckTolerance) + { + + uint embeddedSessionId = udpPayload.ReadUInt32LittleEndian(20); + if (embeddedSessionId == 0) + embeddedSessionId = sessionId; + if (!wneStories.ContainsKey(embeddedSessionId)) + { + byte embeddedPacketType = udpPayload[17]; + if (embeddedPacketType == 0x03) + { + wneStories.Add(embeddedSessionId, new WneStory(embeddedSessionId)); + _logger.InfoFormat("Found new embedded WNE story #{0}", embeddedSessionId); + Handler?.OnWneStoryDetect(embeddedSessionId); + } + else + { + OnError("Missed announcement of embedded WNE story #{0}", embeddedSessionId); + Handler?.OnWneStoryFail(outerStory.SessionId); + return false; + } + } + + if (outerStory.timesSucessfullySynced == 0) + { + outerStory.ExpectedEccGroup = udpPayload[8]; + outerStory.ExpectedContinuityCounter = udpPayload[11] + 1; + outerStory.timesSucessfullySynced++; + } + else + { + if (outerStory.ExpectedEccGroup != udpPayload[8]) + { + OnError("Expected ECC group {0} but got {1}", outerStory.ExpectedEccGroup, udpPayload[8]); + Handler?.OnWneStoryFail(outerStory.SessionId); + return false; + } + + if (udpPayload[0x000b] == 0x07 && udpPayload[0x000c] == 0x03) + { + outerStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1); + outerStory.ExpectedContinuityCounter = 0; + return true; + } + + bool isFileFinished = udpPayload[18] == 0xff; + + if (udpPayload[11] != 0x07 || udpPayload[12] != 0x04) + { + if (isFileFinished) + { + if (udpPayload[12] != 0x01) + { + OnError("Expected Continuity Counter {0} but got {1}", outerStory.ExpectedContinuityCounter, udpPayload[11]); + Handler?.OnWneStoryFail(outerStory.SessionId); + return false; + } + + outerStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1); + outerStory.ExpectedContinuityCounter = 0; + return true; + } + else + { + outerStory.ExpectedEccGroup = udpPayload[8]; + outerStory.ExpectedContinuityCounter = udpPayload[11] + 1; + } + } + else + { + outerStory.ExpectedEccGroup = udpPayload[8]; + outerStory.ExpectedContinuityCounter = udpPayload[11] + 1; + outerStory.timesSucessfullySynced++; + } + } + + if (udpPayload[11] == 0x07) + { + outerStory.ExpectedEccGroup = (byte)(udpPayload[8] + 1); + outerStory.ExpectedContinuityCounter = 0; + } + + Span embeddedPayload = udpPayload.Slice(16); + return HandlePacket(embeddedPayload); + + } + else + { + if (udpPayload.Slice(16).IsBlank()) + { + return true; + } + + if (udpPayload[12] == 0x05 && udpPayload[14] == 0x09) + { + uint thirdUint = udpPayload.ReadUInt32LittleEndian(8); + if (thirdUint != wneStories[sessionId].ExpectedPayloadBlock) + { + OnError("Expected payload block {0} but got {1}. This story ({2}) is incomplete and can not be recovered.", wneStories[sessionId].ExpectedPayloadBlock, thirdUint, sessionId); + wneStories[sessionId].Corrupted = true; + Handler?.OnWneStoryFail(sessionId); + return false; + } + + wneStories[sessionId].AppendPayloadBlock(udpPayload.Slice(16)); + Handler?.OnWneStoryProgress(new WneStoryProgress(wneStories[sessionId])); + return true; + } + + OnError("Unknown packet type in a 0x01 packet."); + return false; + } + } + } + + public bool StopProcessingAfterThis() + { + return true; + } + + private HashSet loggedErrors; + private Dictionary wneStories; + public IWneHandler Handler { get; set; } + + private void OnError(string message, params object[] args) + { + if (loggedErrors == null) + loggedErrors = new HashSet(); + + string unpacked = String.Format(message, args); + if (!loggedErrors.Contains(unpacked)) + { loggedErrors.Add(unpacked); unpacked = String.Format("Packet #{0}: {1}", packetSerial, unpacked); - _logger.Warn(unpacked); - } - } - - -} + _logger.Warn(unpacked); + } + } + + +}