diff --git a/GUIs/skyscraper8.UI.ImGui/Forms/JobDisplay.cs b/GUIs/skyscraper8.UI.ImGui/Forms/JobDisplay.cs index 654e40d..7b21a45 100644 --- a/GUIs/skyscraper8.UI.ImGui/Forms/JobDisplay.cs +++ b/GUIs/skyscraper8.UI.ImGui/Forms/JobDisplay.cs @@ -8,11 +8,14 @@ using System.Reflection.Metadata.Ecma335; using System.Text; using System.Threading.Tasks; using ImGuiNET; +using moe.yo3explorer.skyscraper8.DVBI.Model; using SDL2Demo.Net; using skyscraper5.Docsis; +using skyscraper5.Dvb.DataBroadcasting.IntModel; using skyscraper5.Dvb.DataBroadcasting.SkyscraperVfs; using skyscraper5.Dvb.Descriptors; using skyscraper5.Dvb.Psi.Model; +using skyscraper5.Dvb.SystemSoftwareUpdate.Model; using skyscraper5.Mhp; using skyscraper5.Mhp.Descriptors; using skyscraper5.Mhp.Si; @@ -27,9 +30,17 @@ using skyscraper5.Skyscraper.Net; using skyscraper5.Skyscraper.Scraper; using skyscraper5.src.Skyscraper.FrequencyListGenerator; using skyscraper5.Teletext.Wss; +using skyscraper8.DvbNip; +using skyscraper8.DvbNip.UiIntegration; +using skyscraper8.GSE; +using skyscraper8.GSE.GSE; +using skyscraper8.Ietf.FLUTE; +using skyscraper8.Ses; using skyscraper8.Skyscraper.Drawing; using skyscraper8.Skyscraper.FrequencyListGenerator; +using skyscraper8.Skyscraper.Scraper; using testdrid.SdlWrapper; +using Platform = skyscraper5.Dvb.DataBroadcasting.IntModel.Platform; namespace SDL2Demo.Forms { @@ -550,7 +561,12 @@ namespace SDL2Demo.Forms patEntry.pmt = result; } - + + public void NotifyMpeTraffic(IpTrafficInfo iti, byte[] ipv4PacketLength) + { + throw new NotImplementedException(); + } + private string pmtTabBarUuid; public void RenderPmt() { @@ -1151,7 +1167,12 @@ namespace SDL2Demo.Forms dsmCcDisplay.Remove(id); } } - + + public void NotifyWss(ushort programNumber, WssDataBlock wssDataBlock, int pid) + { + throw new NotImplementedException(); + } + private void RenderDataCarousels() { if (dsmCcDisplay == null) @@ -2182,7 +2203,107 @@ namespace SDL2Demo.Forms throw new NotImplementedException(); } - public TaskQueue Tasks { get; set; } + public void NotifyNit(NitNetwork nitNetwork) + { + throw new NotImplementedException(); + } + + public void EnableUiFeature(SkyscraperUiFeature bbframeAnalysis) + { + throw new NotImplementedException(); + } + + public void OnIpMacNotification(int sourcePid, Platform platform, Target target, Operational operational) + { + throw new NotImplementedException(); + } + + public void OnSsuNotification(UpdateNotificationGroup common, UpdateNotificationTarget target, ushort programNumber) + { + throw new NotImplementedException(); + } + + public void OnBbframe(BBHeader bbHeader, byte[] payload) + { + throw new NotImplementedException(); + } + + public void OnDetectionOfInnerTs(SkyscraperContext child, object identifier) + { + throw new NotImplementedException(); + } + + public void NotifyGsePacket(ushort value, byte[] gseDataBytes, GseLabel label) + { + throw new NotImplementedException(); + } + + public void OnDvbNipFileArrival(FluteUiHandle fuh) + { + throw new NotImplementedException(); + } + + public void OnDvbNipMulticastGatewayConfiguration(NipActualCarrierInformation carrier, + MulticastGatewayConfigurationType multicastGatewayConfiguration) + { + throw new NotImplementedException(); + } + + public void OnDvbNipCarrierDetected(NipActualCarrierInformation currentCarrierInformation) + { + throw new NotImplementedException(); + } + + public void OnDvbNipPrivateDataSignallingManifest(PrivateDataSignallingManifestType privateDataSignallingManifest) + { + throw new NotImplementedException(); + } + + public void OnDvbNipServiceListEntryPoints(NipActualCarrierInformation currentCarrierInformation, + ServiceListEntryPoints serviceListEntryPoints, DateTime dvbNipTime) + { + throw new NotImplementedException(); + } + + public void OnDvbNipServiceList(NipActualCarrierInformation currentCarrierInformation, string serviceListId1, + string serviceListId2) + { + throw new NotImplementedException(); + } + + public void OnDvbNipTimeOffsetFile(NipActualCarrierInformation currentCarrierInformation, TimeOffsetFileType timeOffsetFile) + { + throw new NotImplementedException(); + } + + public void OnDvbNipNetworkInformationFile(NipActualCarrierInformation currentCarrierInformation, + NetworkInformationFileType networkInformationFile) + { + throw new NotImplementedException(); + } + + public void DvbNipServiceInformation(NipActualCarrierInformation currentCarrierInformation, + ServiceInformationFileType serviceInformationFile) + { + throw new NotImplementedException(); + } + + public void OnDvbNipFileAnnouncement(FDTInstanceType flute) + { + throw new NotImplementedException(); + } + + public void OnAstraSgtList(SgtList list) + { + throw new NotImplementedException(); + } + + public void OnAstraSgtService(SgtService child) + { + throw new NotImplementedException(); + } + + public TaskQueue Tasks { get; set; } private ushort? ResolveCaId(ProgramMapping patValuePmt, SdtCoordinate sdt) { diff --git a/GUIs/skyscraper8.UI.ImGui/Jobs/InheritedBlindscanUiJunction.cs b/GUIs/skyscraper8.UI.ImGui/Jobs/InheritedBlindscanUiJunction.cs index 85a059c..9f8c38a 100644 --- a/GUIs/skyscraper8.UI.ImGui/Jobs/InheritedBlindscanUiJunction.cs +++ b/GUIs/skyscraper8.UI.ImGui/Jobs/InheritedBlindscanUiJunction.cs @@ -33,8 +33,19 @@ using System.Numerics; using System.Text; using System.Threading.Tasks; using System.Xml.Serialization; +using moe.yo3explorer.skyscraper8.DVBI.Model; +using skyscraper5.Dvb.DataBroadcasting.IntModel; +using skyscraper5.Dvb.SystemSoftwareUpdate.Model; +using skyscraper8.DvbNip; +using skyscraper8.DvbNip.UiIntegration; +using skyscraper8.GSE; +using skyscraper8.GSE.GSE; +using skyscraper8.Ietf.FLUTE; +using skyscraper8.Ses; +using skyscraper8.Skyscraper.Scraper; using testdrid.SdlWrapper; using static SDL2Demo.Jobs.Blindscan; +using Platform = skyscraper5.Dvb.DataBroadcasting.IntModel.Platform; namespace SDL2Demo.Jobs { @@ -452,6 +463,11 @@ namespace SDL2Demo.Jobs patEntry.pmt = result; } + public void NotifyMpeTraffic(IpTrafficInfo iti, byte[] ipv4PacketLength) + { + throw new NotImplementedException(); + } + private string pmtTabBarUuid; public void RenderPmt() @@ -996,6 +1012,11 @@ namespace SDL2Demo.Jobs } } + public void NotifyWss(ushort programNumber, WssDataBlock wssDataBlock, int pid) + { + throw new NotImplementedException(); + } + private void RenderDataCarousels() { @@ -1800,6 +1821,106 @@ namespace SDL2Demo.Jobs return !foundFrequenciesWindow.doNotAutoZap; } + public void NotifyNit(NitNetwork nitNetwork) + { + throw new NotImplementedException(); + } + + public void EnableUiFeature(SkyscraperUiFeature bbframeAnalysis) + { + throw new NotImplementedException(); + } + + public void OnIpMacNotification(int sourcePid, Platform platform, Target target, Operational operational) + { + throw new NotImplementedException(); + } + + public void OnSsuNotification(UpdateNotificationGroup common, UpdateNotificationTarget target, ushort programNumber) + { + throw new NotImplementedException(); + } + + public void OnBbframe(BBHeader bbHeader, byte[] payload) + { + throw new NotImplementedException(); + } + + public void OnDetectionOfInnerTs(SkyscraperContext child, object identifier) + { + throw new NotImplementedException(); + } + + public void NotifyGsePacket(ushort value, byte[] gseDataBytes, GseLabel label) + { + throw new NotImplementedException(); + } + + public void OnDvbNipFileArrival(FluteUiHandle fuh) + { + throw new NotImplementedException(); + } + + public void OnDvbNipMulticastGatewayConfiguration(NipActualCarrierInformation carrier, + MulticastGatewayConfigurationType multicastGatewayConfiguration) + { + throw new NotImplementedException(); + } + + public void OnDvbNipCarrierDetected(NipActualCarrierInformation currentCarrierInformation) + { + throw new NotImplementedException(); + } + + public void OnDvbNipPrivateDataSignallingManifest(PrivateDataSignallingManifestType privateDataSignallingManifest) + { + throw new NotImplementedException(); + } + + public void OnDvbNipServiceListEntryPoints(NipActualCarrierInformation currentCarrierInformation, + ServiceListEntryPoints serviceListEntryPoints, DateTime dvbNipTime) + { + throw new NotImplementedException(); + } + + public void OnDvbNipServiceList(NipActualCarrierInformation currentCarrierInformation, string serviceListId1, + string serviceListId2) + { + throw new NotImplementedException(); + } + + public void OnDvbNipTimeOffsetFile(NipActualCarrierInformation currentCarrierInformation, TimeOffsetFileType timeOffsetFile) + { + throw new NotImplementedException(); + } + + public void OnDvbNipNetworkInformationFile(NipActualCarrierInformation currentCarrierInformation, + NetworkInformationFileType networkInformationFile) + { + throw new NotImplementedException(); + } + + public void DvbNipServiceInformation(NipActualCarrierInformation currentCarrierInformation, + ServiceInformationFileType serviceInformationFile) + { + throw new NotImplementedException(); + } + + public void OnDvbNipFileAnnouncement(FDTInstanceType flute) + { + throw new NotImplementedException(); + } + + public void OnAstraSgtList(SgtList list) + { + throw new NotImplementedException(); + } + + public void OnAstraSgtService(SgtService child) + { + throw new NotImplementedException(); + } + public TaskQueue Tasks { get; set; } internal JobContext jobContext; diff --git a/skyscraper8/Abertis/AbertisDecoder.cs b/skyscraper8/Abertis/AbertisDecoder.cs index d1128c8..b19bbfb 100644 --- a/skyscraper8/Abertis/AbertisDecoder.cs +++ b/skyscraper8/Abertis/AbertisDecoder.cs @@ -26,14 +26,10 @@ namespace skyscraper5.Abertis public void PushPacket(TsPacket packet) { - byte[] packetPayload = packet.Payload; - - MemoryStream ms = new MemoryStream(packetPayload); - if (packet.AdaptionFieldControl == 3) - { - ms.ReadByte(); - } + Span payload = packet.GetPayload(); + MemoryStream ms = new MemoryStream(payload.ToArray(), false); + int syncLossCheck = 0; while (ms.GetAvailableBytes() > 0) { diff --git a/skyscraper8/Docsis/DocsisPacketProcessor.cs b/skyscraper8/Docsis/DocsisPacketProcessor.cs index 97af930..398bb4c 100644 --- a/skyscraper8/Docsis/DocsisPacketProcessor.cs +++ b/skyscraper8/Docsis/DocsisPacketProcessor.cs @@ -88,21 +88,24 @@ namespace skyscraper5.Docsis public void PushPacket(TsPacket packet) { - if (packet.AdaptionFieldControl != 1) + TsAdaptionField adaptionField = packet.AdaptionField; + Span payload = packet.GetPayload(); + + if (adaptionField != null || payload == null) { PacketLoss(); return; } bool packetPayloadUnitStart = packet.PayloadUnitStart; - MemoryStream ms = new MemoryStream(packet.Payload, false); + MemoryStream ms = new MemoryStream(payload.ToArray(), false); if (!currentPayloadUnitPresent && !packetPayloadUnitStart) return; if (!currentPayloadUnitPresent && packetPayloadUnitStart) { - ms.Position = packet.PayloadStartOffset; + ms.Position = packet.PID; currentPayloadUnitPresent = true; currentPayloadBuffer = new MemoryStream(); } diff --git a/skyscraper8/Dvb/DataBroadcasting/IntModel/IpMacNotification.cs b/skyscraper8/Dvb/DataBroadcasting/IntModel/IpMacNotification.cs index f547913..b62500d 100644 --- a/skyscraper8/Dvb/DataBroadcasting/IntModel/IpMacNotification.cs +++ b/skyscraper8/Dvb/DataBroadcasting/IntModel/IpMacNotification.cs @@ -9,114 +9,32 @@ namespace skyscraper5.Dvb.DataBroadcasting.IntModel { public class IpMacNotification { - public IpMacNotification(Platform platform, int targetType, string targetName, Operational operational) + public IpMacNotification(Platform platform, Target target, Operational operational) + { + Platform = platform; + Target = target; + Operational = operational; + } + + public Platform Platform { get; } + public Target Target { get; } + public Operational Operational { get; } + + public static IEnumerable FlatMap(Platform platform, Target target, Operational operational) { - PlatformName = platform.Name; - PlatformProviderName = platform.ProviderName; - PlatformProviderNameLanguageCode = platform.ProviderNameLanguageCode; - PlatformActionType = platform.ActionType; - PlatformNameLanguageCode = platform.NameLanguageCode; - PlatformId = platform.PlatformId; - PlatformProcessingOrder = platform.ProcessingOrder; - - TargetType = targetType; - TargetName = targetName; - - OperationalNetworkId = operational.NetworkId; - OperationalMpeFecAlgorithm = operational.MpeFecAlgorithm; - OperationalComponentTag = operational.ComponentTag; - OperationalTransportStreamId = operational.TransportStreamId; - OperationalTimeSliceFecId = operational.TimeSliceFecId; - OperationalTimeSlicing = operational.TimeSlicing; - OperationalOriginalNetworkId = operational.OriginalNetworkId; - OperationalServiceId = operational.ServiceId; - OperationalTimeSliceFrameSize = operational.TimeSliceFrameSize; - OperationalTimeSliceMaxAverageRate = operational.TimeSliceMaxAverageRate; - OperationalTimeSliceMaxBurstDuration = operational.TimeSliceMaxBurstDuration; - + yield return new IpMacNotification(platform, target, operational); } - public byte? OperationalTimeSliceMaxBurstDuration { get; set; } + public override bool Equals(object? obj) + { + return obj is IpMacNotification notification && + EqualityComparer.Default.Equals(Platform, notification.Platform) && + EqualityComparer.Default.Equals(Target, notification.Target); + } - public int? OperationalTimeSliceMaxAverageRate { get; set; } - - public int? OperationalTimeSliceFrameSize { get; set; } - - public ushort? OperationalServiceId { get; set; } - - public ushort? OperationalOriginalNetworkId { get; set; } - - public bool? OperationalTimeSlicing { get; set; } - - public int? OperationalTimeSliceFecId { get; set; } - - public ushort? OperationalTransportStreamId { get; set; } - - public byte? OperationalComponentTag { get; set; } - - public int? OperationalMpeFecAlgorithm { get; set; } - - public ushort? OperationalNetworkId { get; set; } - - public string TargetName { get; set; } - - public int TargetType { get; set; } - - public byte PlatformProcessingOrder { get; set; } - - public uint PlatformId { get; set; } - - public string PlatformNameLanguageCode { get; set; } - - public byte PlatformActionType { get; set; } - - public string PlatformProviderNameLanguageCode { get; set; } - - public string PlatformProviderName { get; set; } - - public string PlatformName { get; set; } - - public static IEnumerable FlatMap(Platform platform, Target target, Operational operational) - { - int results = 0; - - if (target.AllReceivers) - { - yield return new IpMacNotification(platform, -1, null, operational); - yield break; - } - - if (target.TargetIpSlashes != null) - { - foreach (CidrSubnet targetTargetIpSlash in target.TargetIpSlashes) - { - results++; - yield return new IpMacNotification(platform, 1, targetTargetIpSlash.ToString(), operational); - } - } - - if (results == 0) - { - throw new NotImplementedException("Some Targets are not implemented properly."); - } - } - - protected bool Equals(IpMacNotification other) - { - return TargetName == other.TargetName && TargetType == other.TargetType && PlatformId == other.PlatformId; - } - - public override bool Equals(object obj) - { - if (ReferenceEquals(null, obj)) return false; - if (ReferenceEquals(this, obj)) return true; - if (obj.GetType() != this.GetType()) return false; - return Equals((IpMacNotification)obj); - } - - public override int GetHashCode() - { - return HashCode.Combine(TargetName, TargetType, PlatformId); - } - } + public override int GetHashCode() + { + return HashCode.Combine(Platform, Target); + } + } } diff --git a/skyscraper8/Dvb/DataBroadcasting/IntModel/Platform.cs b/skyscraper8/Dvb/DataBroadcasting/IntModel/Platform.cs index fb2b525..5e1ad6a 100644 --- a/skyscraper8/Dvb/DataBroadcasting/IntModel/Platform.cs +++ b/skyscraper8/Dvb/DataBroadcasting/IntModel/Platform.cs @@ -27,5 +27,16 @@ namespace skyscraper5.Dvb.DataBroadcasting.IntModel PlatformId = platformId; ProcessingOrder = processingOrder; } - } + + public override bool Equals(object? obj) + { + return obj is Platform platform && + PlatformId == platform.PlatformId; + } + + public override int GetHashCode() + { + return HashCode.Combine(PlatformId); + } + } } diff --git a/skyscraper8/Dvb/DataBroadcasting/IntModel/Target.cs b/skyscraper8/Dvb/DataBroadcasting/IntModel/Target.cs index edd761a..f5a3871 100644 --- a/skyscraper8/Dvb/DataBroadcasting/IntModel/Target.cs +++ b/skyscraper8/Dvb/DataBroadcasting/IntModel/Target.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net; using System.Text; using System.Threading.Tasks; using skyscraper5.Skyscraper.Net; @@ -14,5 +15,46 @@ namespace skyscraper5.Dvb.DataBroadcasting.IntModel //Was Loop empty? public bool AllReceivers { get; set; } - } + + public override bool Equals(object? obj) + { + return obj is Target target && + EqualityComparer>.Default.Equals(TargetIpSlashes, target.TargetIpSlashes) && + AllReceivers == target.AllReceivers; + } + + public override int GetHashCode() + { + return HashCode.Combine(TargetIpSlashes, AllReceivers); + } + + public uint GenerateId() + { + if (AllReceivers) + return uint.MaxValue; + + foreach(CidrSubnet subnet in TargetIpSlashes) + { + IPAddress ipAddress = subnet.IpAddress; + byte[] bytes = ipAddress.GetAddressBytes(); + return BitConverter.ToUInt32(bytes); + } + + throw new NotImplementedException(String.Format("Failed to derive an identifier for a IP/MAC Notification target. This is a bug, the developers would need a sample of this TS to fix this.")); + } + + public override string ToString() + { + if (AllReceivers) + return ""; + + StringBuilder sb = new StringBuilder(); + foreach (CidrSubnet subnet in TargetIpSlashes) + { + sb.Append(subnet.ToString()); + sb.Append(","); + } + return sb.ToString(); + } + } } diff --git a/skyscraper8/GS/Stid135BbFrameReader.cs b/skyscraper8/GS/Stid135BbFrameReader.cs index e8e8e09..960b71e 100644 --- a/skyscraper8/GS/Stid135BbFrameReader.cs +++ b/skyscraper8/GS/Stid135BbFrameReader.cs @@ -30,7 +30,7 @@ namespace skyscraper8.GSE private bool annoncementDone; public void PushPacket(TsPacket packet) { - byte[] packets = packet.RawPacket; + byte[] packets = packet.GetRawPacket(); int pid = packets[1]; pid &= 0x01f; pid <<= 8; diff --git a/skyscraper8/Ietf/Rfc4236_ULE/UlePacketProcessor.cs b/skyscraper8/Ietf/Rfc4236_ULE/UlePacketProcessor.cs index e7d3aea..b621584 100644 --- a/skyscraper8/Ietf/Rfc4236_ULE/UlePacketProcessor.cs +++ b/skyscraper8/Ietf/Rfc4236_ULE/UlePacketProcessor.cs @@ -30,7 +30,9 @@ namespace skyscraper8.Ietf.Rfc4236_ULE public void PushPacket(TsPacket packet) { - if (packet.AdaptionFieldControl != 1) + TsAdaptionField packetAdaptionField = packet.AdaptionField; + Span packetPayload = packet.GetPayload(); + if (packetAdaptionField != null || packetPayload == null) { _eventHandler.OnUleError(_pid, 2); return; @@ -44,7 +46,7 @@ namespace skyscraper8.Ietf.Rfc4236_ULE if (reassemblyMode) { int availableBytes = Math.Min(184, reassemblyRemainingBytes); - reassemblyStream.Write(packet.RawPacket, 4, availableBytes); + reassemblyStream.Write(packetPayload.ToArray(), 4, availableBytes); reassemblyRemainingBytes -= availableBytes; if (reassemblyRemainingBytes == 0) { @@ -68,7 +70,7 @@ namespace skyscraper8.Ietf.Rfc4236_ULE { bool multipleSndusInPacket = false; - MemoryStream payload = new MemoryStream(packet.RawPacket, false); + MemoryStream payload = new MemoryStream(packet.GetRawPacket(), false); payload.Position += 4; byte nextPacketOffset = payload.ReadUInt8(); if (!reassemblyMode) diff --git a/skyscraper8/Json/IpAddressJsonConverter.cs b/skyscraper8/Json/IpAddressJsonConverter.cs new file mode 100644 index 0000000..c1a720b --- /dev/null +++ b/skyscraper8/Json/IpAddressJsonConverter.cs @@ -0,0 +1,24 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.Json +{ + public class IpAddressJsonConverter : JsonConverter + { + public override IPAddress? ReadJson(JsonReader reader, Type objectType, IPAddress? existingValue, bool hasExistingValue, JsonSerializer serializer) + { + string s = (string)reader.Value; + return IPAddress.Parse(s); + } + + public override void WriteJson(JsonWriter writer, IPAddress? value, JsonSerializer serializer) + { + writer.WriteValue(value.ToString()); + } + } +} diff --git a/skyscraper8/Mpeg2/PacketFilter/BissDescrambleFilter.cs b/skyscraper8/Mpeg2/PacketFilter/BissDescrambleFilter.cs index 69195a0..d4146a7 100644 --- a/skyscraper8/Mpeg2/PacketFilter/BissDescrambleFilter.cs +++ b/skyscraper8/Mpeg2/PacketFilter/BissDescrambleFilter.cs @@ -52,8 +52,9 @@ namespace skyscraper8.Mpeg2.PacketFilter } int retsize = 0; - byte[] unscrambled = new byte[packet.Payload.Length]; - csa2.DecryptImpl(packet.Payload, packet.Payload.Length, unscrambled, unscrambled.Length, ref retsize); + Span scrambledPayload = packet.GetPayload(); + byte[] unscrambled = new byte[scrambledPayload.Length]; + csa2.DecryptImpl(scrambledPayload, scrambledPayload.Length, unscrambled, unscrambled.Length, ref retsize); packet.SetUnscrambled(unscrambled); return true; diff --git a/skyscraper8/Mpeg2/PacketFilter/TsRecorder.cs b/skyscraper8/Mpeg2/PacketFilter/TsRecorder.cs index a496706..ee345e1 100644 --- a/skyscraper8/Mpeg2/PacketFilter/TsRecorder.cs +++ b/skyscraper8/Mpeg2/PacketFilter/TsRecorder.cs @@ -20,7 +20,8 @@ namespace skyscraper5.src.Mpeg2.PacketFilter { if (recordingBufferedStream != null) { - recordingBufferedStream.Write(packet.RawPacket, 0, packet.RawPacket.Length); + byte[] rawPacket = packet.GetRawPacket(); + recordingBufferedStream.Write(rawPacket, 0, rawPacket.Length); } return true; } diff --git a/skyscraper8/Mpeg2/PcrMonitor.cs b/skyscraper8/Mpeg2/PcrMonitor.cs index a8aff63..b84022e 100644 --- a/skyscraper8/Mpeg2/PcrMonitor.cs +++ b/skyscraper8/Mpeg2/PcrMonitor.cs @@ -11,16 +11,16 @@ namespace skyscraper5.Mpeg2 internal void Analyze(TsPacket packet) { - if (packet.Adaption == null) + if (packet.AdaptionField == null) return; - if (!packet.Adaption.Valid) + if (!packet.AdaptionField.Valid) return; - if (!packet.Adaption.PcrPresent) + if (!packet.AdaptionField.PcrPresent) return; if (StartPcr == null) { - StartPcr = packet.Adaption.PCR; + StartPcr = packet.AdaptionField.PCR; SourcePid = packet.PID; StartRealTime = DateTime.Now; } @@ -28,7 +28,7 @@ namespace skyscraper5.Mpeg2 { if (packet.PID == SourcePid) { - LastPcr = packet.Adaption.PCR; + LastPcr = packet.AdaptionField.PCR; LastRealTime = DateTime.Now; } } diff --git a/skyscraper8/Mpeg2/PesDecoder.cs b/skyscraper8/Mpeg2/PesDecoder.cs index 2fc95b4..9414b14 100644 --- a/skyscraper8/Mpeg2/PesDecoder.cs +++ b/skyscraper8/Mpeg2/PesDecoder.cs @@ -18,7 +18,7 @@ namespace skyscraper5.Mpeg2 public void PushPacket(TsPacket packet) { - if (packet.Payload == null) + if (packet.GetPayload() == null) return; pushed++; @@ -32,7 +32,8 @@ namespace skyscraper5.Mpeg2 } ms = new MemoryStream(); ms.WriteByte(0); - ms.Write(packet.Payload, 0, packet.Payload.Length); + Span payload = packet.GetPayload(); + ms.Write(payload.ToArray(), 0, payload.Length); } else @@ -44,7 +45,8 @@ namespace skyscraper5.Mpeg2 } else { - ms.Write(packet.Payload, 0, packet.Payload.Length); + Span payload = packet.GetPayload(); + ms.Write(payload.ToArray(), 0, payload.Length); } } } diff --git a/skyscraper8/Mpeg2/PsiDecoder.cs b/skyscraper8/Mpeg2/PsiDecoder.cs index afa6775..7075378 100644 --- a/skyscraper8/Mpeg2/PsiDecoder.cs +++ b/skyscraper8/Mpeg2/PsiDecoder.cs @@ -18,7 +18,7 @@ namespace skyscraper5.Mpeg2 public void PushPacket(TsPacket packet) { - if (packet.Payload == null) + if (packet.GetPayload() == null) return; if (packet.PID != pid) return; @@ -40,11 +40,17 @@ namespace skyscraper5.Mpeg2 } } - MemoryStream payload = new MemoryStream(packet.Payload); + MemoryStream payload = new MemoryStream(packet.GetPayload().ToArray(), false); long available = payload.Length - payload.Position; if (psiJustStarted) { - long startOffset = packet.PayloadStartOffset; + payload.Position = 0; + byte payloadUnitStartOffset = payload.ReadUInt8(); + available--; + payload.Position += payloadUnitStartOffset; + available -= payloadUnitStartOffset; + + long startOffset = 0; if (startOffset > 0) { startOffset = payload.Position + startOffset; @@ -102,6 +108,11 @@ namespace skyscraper5.Mpeg2 payload.Position--; if (payloadPos != 0xff) newSection = true; + if (payload.Position == 0) + { + payload.Position++; + available--; + } } if (newSection) diff --git a/skyscraper8/Mpeg2/TsAdaptionField.cs b/skyscraper8/Mpeg2/TsAdaptionField.cs index 8a274cc..7120792 100644 --- a/skyscraper8/Mpeg2/TsAdaptionField.cs +++ b/skyscraper8/Mpeg2/TsAdaptionField.cs @@ -5,28 +5,24 @@ using skyscraper5.Skyscraper.IO; namespace skyscraper5.Mpeg2 { - public class DvbAdaptionField : Validatable + public class TsAdaptionField : Validatable { - public DvbAdaptionField(Stream binaryReader, bool TEI) + public TsAdaptionField(Span adaptionField, bool TEI) { - byte adaptionFieldLength = binaryReader.ReadUInt8(); + byte adaptionFieldLength = (byte)adaptionField.Length; if (adaptionFieldLength > 183) { Valid = false; return; } - byte[] adaptionField = binaryReader.ReadBytes(adaptionFieldLength); - RawAdaptionField = adaptionField; - binaryReader = new MemoryStream(adaptionField); - this.Length = adaptionFieldLength; - + if (adaptionFieldLength == 0) { Valid = true; return; //Seems valid according to ISO 13818-1 } - byte bitmask = binaryReader.ReadUInt8(); + byte bitmask = adaptionField[0]; Discontinuity = ((bitmask & 0x80) >> 7) != 0; RandomAccess = ((bitmask & 0x40) >> 6) != 0; PriorityIndicator = ((bitmask & 0x20) >> 5) != 0; @@ -36,6 +32,7 @@ namespace skyscraper5.Mpeg2 TransportPrivateDataPresent = ((bitmask & 0x02) >> 1) != 0; AdaptionFieldExtensionPresent = ((bitmask & 0x01)) != 0; + MemoryStream binaryReader = new MemoryStream(adaptionField.ToArray(), 1, adaptionFieldLength - 1); if (PcrPresent) { if (binaryReader.GetAvailableBytes() < 6) @@ -179,6 +176,5 @@ namespace skyscraper5.Mpeg2 public bool PcrPresent { get; private set; } public byte Length { get; private set; } - public byte[] RawAdaptionField { get; private set; } } } diff --git a/skyscraper8/Mpeg2/TsContext.cs b/skyscraper8/Mpeg2/TsContext.cs index 5fdba55..15845f9 100644 --- a/skyscraper8/Mpeg2/TsContext.cs +++ b/skyscraper8/Mpeg2/TsContext.cs @@ -123,7 +123,7 @@ namespace skyscraper5.Mpeg2 private bool EnsureContinuity(TsPacket packet) { //Found in ISO 13818-1.pdf, page 38 - if (packet.AdaptionFieldControl == 2 || packet.AdaptionFieldControl == 0) + if (packet.GetPayload() != null) return true; if (packet.PID == 0x1fff) @@ -295,11 +295,11 @@ namespace skyscraper5.Mpeg2 [MethodImpl(MethodImplOptions.AggressiveInlining)] private void InjectIntoPcrMonitor(TsPacket tspacket) { - if (tspacket.Adaption == null) + if (tspacket.AdaptionField == null) return; - if (!tspacket.Adaption.Valid) + if (!tspacket.AdaptionField.Valid) return; - if (!tspacket.Adaption.PcrPresent) + if (!tspacket.AdaptionField.PcrPresent) return; if (PcrMonitor != null) diff --git a/skyscraper8/Mpeg2/TsPacket.cs b/skyscraper8/Mpeg2/TsPacket.cs index 7d94cd7..a076356 100644 --- a/skyscraper8/Mpeg2/TsPacket.cs +++ b/skyscraper8/Mpeg2/TsPacket.cs @@ -9,124 +9,78 @@ namespace skyscraper5.Mpeg2 { public TsPacket(byte[] buffer) { - Serial = serialGen++; - - RawPacket = (byte[])buffer.Clone(); if (buffer.Length != 188) - throw new ArgumentException("buffer.Length != 188"); - - MemoryStream br = new MemoryStream(buffer, false); - uint anInt = br.ReadUInt32BE(); - uint syncBytes = (anInt & 0xff000000) >> 24; - if (syncBytes != 'G') - { - throw new InvalidTsPacketException(); - } - - TEI = (((anInt & 0x800000)) >> 23) != 0; - PayloadUnitStart = (((anInt & 0x400000)) >> 22) != 0; - Priority = ((anInt & 0x200000)) >> 21; - PID = ((anInt & 0x1fff00)) >> 8; - TSC = ((anInt & 0xc0)) >> 6; - AdaptionFieldControl = ((anInt & 0x30)) >> 4; - Continuity = ((anInt & 0xf)); - if (PID == 0x1FFF) - { - return; - } - - if (TEI) - return; - - if ((buffer[3] & 0x20) != 0) - { - //Skip adaption field if present - } - - if ((buffer[1] & 0x40) != 0) - { - //Unit start - } - - PayloadOffset = 4; - if (PayloadUnitStart) - { - PayloadStartOffset = buffer[4] & 0xff; - PayloadOffset++; - } + throw new ArgumentException("Invalid buffer length"); - switch (AdaptionFieldControl) + if (buffer[0] != 0x47) + throw new ArgumentException("Buffer is not a TS packet"); + + rawPacket = buffer; + + TEI = (buffer[1] & 0x80) != 0; + PayloadUnitStart = (buffer[1] & 0x40) != 0; + TransportPriority = (buffer[1] & 0x20) != 0; + PID = (uint)((buffer[1] & 0x1f) << 8) | buffer[2]; + TSC = ((buffer[3] & 0xc0) >> 6); + int AdaptionFieldControl = ((buffer[3] & 0x30) >> 4); + Continuity = (byte)(buffer[3] & 0x0f); + bool hasAdaptionField = (AdaptionFieldControl & 0x02) != 0; + bool hasPayloadField = (AdaptionFieldControl & 0x01) != 0; + + int ptr = 4; + if (hasAdaptionField) { - case 1: - br.Position = PayloadOffset; - long PayloadLengthB = 188 - PayloadOffset; - Payload = br.ReadBytes(PayloadLengthB); - break; - case 2: - Adaption = new DvbAdaptionField(br, TEI); - if (!Adaption.Valid) - TEI = true; - break; - case 3: - Adaption = new DvbAdaptionField(br, TEI); - if (!Adaption.Valid) - { - TEI = true; - break; - } - PayloadOffset += Adaption.Length; - br.Position = PayloadOffset; - long PayloadLength = (188 - PayloadOffset); - Payload = br.ReadBytes((int)PayloadLength); - break; - default: - TEI = true; - break; + byte adaptionFieldLength = buffer[ptr++]; + int maxLength = 188 - ptr; + if (adaptionFieldLength > maxLength) + { + TEI = true; + return; + } + Span adaptionField = buffer.AsSpan(ptr, adaptionFieldLength); + ptr += adaptionFieldLength; + AdaptionField = new TsAdaptionField(adaptionField, TEI); + } + + if (hasPayloadField) + { + PayloadOffset = ptr; } } - private static ulong serialGen; - public DvbAdaptionField Adaption { get; private set; } + public byte Continuity { get; set; } - public bool TEI { get; private set; } - public bool PayloadUnitStart { get; private set; } - public uint Priority { get; private set; } - public uint PID { get; private set; } + public int TSC { get; set; } - /// - /// 0 = not scrambled - /// 1 = reserved - /// 2 = scrambled, even key - /// 3 = scrambled, odd key - /// - public uint TSC { get; internal set; } - public uint AdaptionFieldControl { get; private set; } - public uint Continuity { get; private set; } - public int PayloadStartOffset { get; private set; } - public byte[] Payload { get; private set; } - public bool PayloadFlagSet => AdaptionFieldControl == 1 || AdaptionFieldControl == 3; - public byte[] RawPacket { get; private set; } - public ulong Serial { get; private set; } + public uint PID { get; set; } - public int PayloadOffset { get; private set; } + public bool TransportPriority { get; set; } - /// - /// Retrieving the raw packet, including adaption field and Payload, but without the four byte header. - /// - /// - public byte[] GetBody() + public bool PayloadUnitStart { get; set; } + + public bool TEI { get; set; } + + public TsAdaptionField AdaptionField { get; set; } + + private int PayloadOffset; + private byte[] rawPacket; + + public Span GetPayload() { - byte[] buffer = new byte[184]; - Array.Copy(RawPacket, 4, buffer, 0, 184); - return buffer; + if (PayloadOffset == 0) + return null; + + return rawPacket.AsSpan(PayloadOffset); } - public void SetUnscrambled(byte[] unscrambledPayload) + public void SetUnscrambled(byte[] unscrambled) { - TSC = 0; - Payload = unscrambledPayload; - Array.Copy(unscrambledPayload, 0, RawPacket, PayloadOffset, unscrambledPayload.Length); - RawPacket[3] &= 0x3f; + throw new NotSupportedException("We aren't doing anything illegal, m'kay?"); + } + + public byte[] GetRawPacket() + { + return rawPacket; } } } diff --git a/skyscraper8/Program.cs b/skyscraper8/Program.cs index aae6c2f..5facd7b 100644 --- a/skyscraper8/Program.cs +++ b/skyscraper8/Program.cs @@ -116,13 +116,16 @@ namespace skyscraper5 { FileInfo fi = new FileInfo(args[1]); FileStream fstream = fi.OpenRead(); - MultiprotocolEncapsulationDecoder mpeDecoder = new MultiprotocolEncapsulationDecoder(null); - PsiDecoder psiDecoder = new PsiDecoder(0x1019, mpeDecoder); + TsContext mpeg2 = new TsContext(); - mpeg2.RegisterPacketProcessor(0x1019, psiDecoder); SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, new InMemoryScraperStorage(), new NullObjectStorage()); skyscraper.InitalizeFilterChain(); - skyscraper.IngestFromStream(fstream); + + MultiprotocolEncapsulationDecoder mpeDecoder = new MultiprotocolEncapsulationDecoder(skyscraper); + PsiDecoder psiDecoder = new PsiDecoder(0x1019, mpeDecoder); + mpeg2.RegisterPacketProcessor(0x1019, psiDecoder); + + skyscraper.IngestFromStream(fstream); } if (args[0].Equals("aactest")) { diff --git a/skyscraper8/Properties/launchSettings.json b/skyscraper8/Properties/launchSettings.json index a02e1d2..01330fc 100644 --- a/skyscraper8/Properties/launchSettings.json +++ b/skyscraper8/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "skyscraper8": { "commandName": "Project", - "commandLineArgs": "bumutest \"C:\\devel\\skyscraper8-testsuite\\12168_pid1019only.ts\"", + "commandLineArgs": "\"C:\\devel\\skyscraper8-testsuite\\express_3928L_t2mi.ts\"", "remoteDebugEnabled": false }, "Container (Dockerfile)": { diff --git a/skyscraper8/Skyscraper/Math/PidEntryCalculatorFilter.cs b/skyscraper8/Skyscraper/Math/PidEntryCalculatorFilter.cs index 58d7817..478b28e 100644 --- a/skyscraper8/Skyscraper/Math/PidEntryCalculatorFilter.cs +++ b/skyscraper8/Skyscraper/Math/PidEntryCalculatorFilter.cs @@ -15,12 +15,12 @@ namespace skyscraper8.Skyscraper.Math if (entropyCalculatorStreams == null) entropyCalculatorStreams = new EntropyCalculatorStream[0x1fff]; - if (packet.PayloadFlagSet) + if (packet.GetPayload() != null) { uint pid = packet.PID; if (entropyCalculatorStreams[pid] == null) entropyCalculatorStreams[pid] = new EntropyCalculatorStream(); - entropyCalculatorStreams[pid].Write(packet.Payload); + entropyCalculatorStreams[pid].Write(packet.GetPayload()); } return true; } diff --git a/skyscraper8/Skyscraper/OldStreamReaderDetector.cs b/skyscraper8/Skyscraper/OldStreamReaderDetector.cs index 54cc6a7..1e871b6 100644 --- a/skyscraper8/Skyscraper/OldStreamReaderDetector.cs +++ b/skyscraper8/Skyscraper/OldStreamReaderDetector.cs @@ -30,7 +30,7 @@ namespace skyscraper5.Skyscraper if (annoncementDone) return; - byte[] packets = packet.RawPacket; + byte[] packets = packet.GetRawPacket(); int pid = (packets[1]); pid &= 0x01f; pid <<= 8; diff --git a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs index bcec7f6..0646128 100644 --- a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs +++ b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs @@ -2226,7 +2226,7 @@ namespace skyscraper5.Skyscraper.Scraper if (!DataStorage.TestForIpMacNotification(notification)) { DataStorage.StoreIpMacNotification(notification); - LogEvent(SkyscraperContextEvent.IpMacNotification, String.Format("{0} -> {1}", platform.Name, notification.TargetName)); + LogEvent(SkyscraperContextEvent.IpMacNotification, String.Format("{0} -> {1}", platform.Name, notification.Target.ToString())); } } } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs index defb3d6..744ad69 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs @@ -721,14 +721,14 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem public bool TestForIpMacNotification(IpMacNotification notification) { - string path = Path.Combine(rootDirectory.FullName, "INT", String.Format("{0}.json", notification.PlatformId)); + string path = Path.Combine(rootDirectory.FullName, "INT", String.Format("{0}_{1}.json", notification.Platform.PlatformId, notification.Target.GenerateId())); return File.Exists(path); } public void StoreIpMacNotification(IpMacNotification notification) { - string path = Path.Combine(rootDirectory.FullName, "INT", String.Format("{0}.json", notification.PlatformId)); - FileInfo fi = new FileInfo(path); + string path = Path.Combine(rootDirectory.FullName, "INT", String.Format("{0}_{1}.json", notification.Platform.PlatformId, notification.Target.GenerateId())); + FileInfo fi = new FileInfo(path); EnsureDirectoryExists(fi.Directory); File.WriteAllText(fi.FullName, JsonConvert.SerializeObject(notification, Formatting.Indented, jsonSerializerSettings)); } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/Utilities/DatabaseKeyInt.cs b/skyscraper8/Skyscraper/Scraper/Storage/Utilities/DatabaseKeyInt.cs index a9d2eec..b197cea 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/Utilities/DatabaseKeyInt.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/Utilities/DatabaseKeyInt.cs @@ -1,41 +1,30 @@ -using System; +using skyscraper5.Dvb.DataBroadcasting.IntModel; +using System; using System.Collections.Generic; namespace skyscraper5.Skyscraper.Scraper.Storage.Utilities { - public class DatabaseKeyInt : IEquatable - { - public DatabaseKeyInt(uint id) - { - Id = id; - } + public class DatabaseKeyInt + { + private uint platformId; + private uint targetId; - public uint Id { get; } + public DatabaseKeyInt(uint platformId, uint v) + { + this.platformId = platformId; + this.targetId = v; + } - public override bool Equals(object obj) - { - return Equals(obj as DatabaseKeyInt); - } + public override bool Equals(object? obj) + { + return obj is DatabaseKeyInt @int && + platformId == @int.platformId && + targetId == @int.targetId; + } - public bool Equals(DatabaseKeyInt other) - { - return other is not null && - Id == other.Id; - } - - public override int GetHashCode() - { - return HashCode.Combine(Id); - } - - public static bool operator ==(DatabaseKeyInt left, DatabaseKeyInt right) - { - return EqualityComparer.Default.Equals(left, right); - } - - public static bool operator !=(DatabaseKeyInt left, DatabaseKeyInt right) - { - return !(left == right); - } - } + public override int GetHashCode() + { + return HashCode.Combine(platformId, targetId); + } + } } diff --git a/skyscraper8/Skyscraper/Scraper/Utils/PacketDumper.cs b/skyscraper8/Skyscraper/Scraper/Utils/PacketDumper.cs index cbe0530..ef7726c 100644 --- a/skyscraper8/Skyscraper/Scraper/Utils/PacketDumper.cs +++ b/skyscraper8/Skyscraper/Scraper/Utils/PacketDumper.cs @@ -22,7 +22,9 @@ namespace skyscraper5.Skyscraper.Scraper.Utils public void PushPacket(TsPacket packet) { - outputFile.Write(packet.RawPacket, 0, packet.RawPacket.Length); + Span payload = packet.GetPayload(); + + outputFile.Write(payload.ToArray(), 0, payload.Length); outputFile.Flush(); } } diff --git a/skyscraper8/Skyscraper/Scraper/Utils/PayloadUnitDumper.cs b/skyscraper8/Skyscraper/Scraper/Utils/PayloadUnitDumper.cs index 4efba84..d5b841f 100644 --- a/skyscraper8/Skyscraper/Scraper/Utils/PayloadUnitDumper.cs +++ b/skyscraper8/Skyscraper/Scraper/Utils/PayloadUnitDumper.cs @@ -39,9 +39,10 @@ namespace skyscraper5.Skyscraper.Scraper.Utils } } } + + Span payload = packet.GetPayload(); targetPs = new MemoryStream(); - targetPs.WriteByte((byte)packet.PayloadStartOffset); - targetPs.Write(packet.Payload, 0, packet.Payload.Length); + targetPs.Write(payload); } else @@ -53,7 +54,8 @@ namespace skyscraper5.Skyscraper.Scraper.Utils } else { - targetPs.Write(packet.Payload, 0, packet.Payload.Length); + Span payload = packet.GetPayload(); + targetPs.Write(payload); } } } diff --git a/skyscraper8/Skyscraper/TcpProxy.cs b/skyscraper8/Skyscraper/TcpProxy.cs index b1e2738..e49740e 100644 --- a/skyscraper8/Skyscraper/TcpProxy.cs +++ b/skyscraper8/Skyscraper/TcpProxy.cs @@ -95,8 +95,8 @@ namespace skyscraper5.Skyscraper try { - byte[] packetRawPacket = packet.RawPacket; - tcpClient.Client.Send(packet.RawPacket); + byte[] packetRawPacket = packet.GetRawPacket(); + tcpClient.Client.Send(packetRawPacket); } catch (SocketException se) { diff --git a/skyscraper8/T2MI/T2MIDecoder.cs b/skyscraper8/T2MI/T2MIDecoder.cs index c58a703..c9c9f11 100644 --- a/skyscraper8/T2MI/T2MIDecoder.cs +++ b/skyscraper8/T2MI/T2MIDecoder.cs @@ -33,20 +33,16 @@ namespace skyscraper5.T2MI private long inputPacketsPushed; private long outputPacketsPushed; private T2MIEventHandler t2MiEventHandler; + private bool firstPacketFound; public void PushPacket(TsPacket packet) { inputPacketsPushed++; bool packetPayloadUnitStart = packet.PayloadUnitStart; - int offset = 4; - if (packet.Adaption != null) - { - offset += packet.Adaption.Length; - offset++; - } + if (!firstPacketFound && packetPayloadUnitStart) + firstPacketFound = true; - MemoryStream ms = new MemoryStream(packet.RawPacket, false); - ms.Position = offset; + MemoryStream ms = new MemoryStream(packet.GetPayload().ToArray(), false); int partsProcessed = 0; byte offsetAccordingToPus = 0; if (packetPayloadUnitStart)