From 8adf5d4ff1f0eb1b7d566960cd7b3543e1f14283 Mon Sep 17 00:00:00 2001 From: feyris-tan <4116042+feyris-tan@users.noreply.github.com> Date: Tue, 10 Mar 2026 22:40:06 +0100 Subject: [PATCH] Patches for Voile Integration --- .../Overrides/Form1UiJunction.cs | 40 +++++++ .../DvbNip/UiIntegration/FluteUiHandle.cs | 30 +++++ skyscraper8/GS/BBframeDeencapsulator3.cs | 2 +- skyscraper8/GS/GSE-BFBS/BfbsGseReader.cs | 5 +- skyscraper8/GS/GSE-HEM/GseHemReader.cs | 6 +- skyscraper8/GS/GSE/GseLabel.cs | 67 ++++++++++++ skyscraper8/GS/GSE/GseReader.cs | 3 + skyscraper8/GS/GsContextDto.cs | 2 + skyscraper8/Mpeg2/TsAdaptionField.cs | 2 + skyscraper8/Program.cs | 13 +++ skyscraper8/Properties/launchSettings.json | 2 +- skyscraper8/Ses/SgtService.cs | 2 +- .../Math/EntropyCalculatorStream.cs | 5 + skyscraper8/Skyscraper/Net/IpTrafficInfo.cs | 10 ++ .../Scraper/ISkyscraperUiJunction.cs | 22 +++- .../Skyscraper/Scraper/SkyscraperContext.cs | 103 ++++++++++++------ .../Skyscraper/Scraper/SkyscraperUiFeature.cs | 4 +- .../Utilities/DatabaseKeyNipService.cs | 39 +++++++ .../Utilities/NipPrivateDataSpecifierDto.cs | 13 +++ .../Contestants/TeletextContestant.cs | 8 +- .../Text/SkyscraperBaseEncoding8.cs | 37 ++++--- skyscraper8/Teletext/ITeletextPageHandler.cs | 4 +- skyscraper8/Teletext/TeletextPesProcessor.cs | 10 +- skyscraper8/skyscraper8.csproj | 6 +- 24 files changed, 369 insertions(+), 66 deletions(-) create mode 100644 skyscraper8/DvbNip/UiIntegration/FluteUiHandle.cs create mode 100644 skyscraper8/Skyscraper/Scraper/Storage/Utilities/DatabaseKeyNipService.cs create mode 100644 skyscraper8/Skyscraper/Scraper/Storage/Utilities/NipPrivateDataSpecifierDto.cs diff --git a/GUIs/skyscraper5.UI/Overrides/Form1UiJunction.cs b/GUIs/skyscraper5.UI/Overrides/Form1UiJunction.cs index b429415..c583340 100644 --- a/GUIs/skyscraper5.UI/Overrides/Form1UiJunction.cs +++ b/GUIs/skyscraper5.UI/Overrides/Form1UiJunction.cs @@ -17,10 +17,15 @@ using System.Linq; using System.Text; using System.Threading.Tasks; using System.Xml.Linq; +using skyscraper5.Dvb.DataBroadcasting.IntModel; +using skyscraper5.Dvb.SystemSoftwareUpdate.Model; using skyscraper5.Skyscraper.Equipment; using skyscraper5.Skyscraper.IO.CrazycatStreamReader; +using skyscraper8.GSE; using skyscraper8.Skyscraper.Drawing; using skyscraper8.Skyscraper.FrequencyListGenerator; +using skyscraper8.Skyscraper.Scraper; +using Platform = skyscraper5.Dvb.DataBroadcasting.IntModel.Platform; namespace skyscraper5.UI.Overrides { @@ -207,8 +212,43 @@ namespace skyscraper5.UI.Overrides throw new NotImplementedException(); } + public void NotifyNit(NitNetwork nitNetwork) + { + throw new NotImplementedException(); + } + + public void EnableUiFeature(SkyscraperUiFeature bbframeAnalysis) + { + throw new NotImplementedException(); + } + + public void OnIpMacNotification(int sourcePid, Platform platform, Target target, Operational operational) + { + throw new NotImplementedException(); + } + + public void OnSsuNotification(UpdateNotificationGroup common, UpdateNotificationTarget target, ushort programNumber) + { + throw new NotImplementedException(); + } + + public void OnBbframe(BBHeader bbHeader, byte[] payload) + { + throw new NotImplementedException(); + } + + public void OnDetectionOfInnerTs(SkyscraperContext child, object identifier) + { + throw new NotImplementedException(); + } + public TaskQueue Tasks { get; set; } + public void NotifyMpeTraffic(IpTrafficInfo iti, byte[] ipv4PacketLength) + { + throw new NotImplementedException(); + } + public void NotifyAit(AitApplication aitApplication) { SkyscraperUiNode applicationNode = form1.EnsureNodeExists("AIT", aitApplication.TryGetName()); diff --git a/skyscraper8/DvbNip/UiIntegration/FluteUiHandle.cs b/skyscraper8/DvbNip/UiIntegration/FluteUiHandle.cs new file mode 100644 index 0000000..357740b --- /dev/null +++ b/skyscraper8/DvbNip/UiIntegration/FluteUiHandle.cs @@ -0,0 +1,30 @@ +using skyscraper8.Ietf.FLUTE; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.DvbNip.UiIntegration +{ + public class FluteUiHandle + { + public FluteUiHandle(NipActualCarrierInformation carrier, FluteListener listener) + { + this.Carrier = carrier; + this.DestinationAddress = listener.DestinationAddress; + this.DestinationPort = listener.DestinationPort; + this.DestinationTsi = listener.DestinationTsi; + this.DestinationToi = listener.DestinationToi; + this.FileAssociation = listener.FileAssociation; + } + + public NipActualCarrierInformation Carrier { get; } + public IPAddress DestinationAddress { get; } + public ushort DestinationPort { get; } + public ulong DestinationTsi { get; } + public ulong DestinationToi { get; } + public FileType FileAssociation { get; } + } +} diff --git a/skyscraper8/GS/BBframeDeencapsulator3.cs b/skyscraper8/GS/BBframeDeencapsulator3.cs index 079efdb..8d05edb 100644 --- a/skyscraper8/GS/BBframeDeencapsulator3.cs +++ b/skyscraper8/GS/BBframeDeencapsulator3.cs @@ -41,7 +41,7 @@ class BbframeDeencapsulator3 : IBbframeDeencapsulator } mis[bbHeader.Matype2].PushFrame(bbHeader, new ReadOnlySpan(bbframe, 11, bbframe.Length - 11)); - context.UiJunction.OnBbframe(bbHeader, bbframe); + context.UiJunction?.OnBbframe(bbHeader, bbframe); } private IMisHandler[] mis; diff --git a/skyscraper8/GS/GSE-BFBS/BfbsGseReader.cs b/skyscraper8/GS/GSE-BFBS/BfbsGseReader.cs index de21294..430f4a7 100644 --- a/skyscraper8/GS/GSE-BFBS/BfbsGseReader.cs +++ b/skyscraper8/GS/GSE-BFBS/BfbsGseReader.cs @@ -1,6 +1,7 @@ using skyscraper5.Dvb.DataBroadcasting; using skyscraper5.Mpeg2; using skyscraper5.Skyscraper.IO; +using skyscraper5.src.InteractionChannel; using skyscraper8.GSE; using skyscraper8.GSE.GSE; using skyscraper8.Skyscraper.IO; @@ -11,7 +12,7 @@ using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; -using skyscraper5.src.InteractionChannel; +using static skyscraper5.Dvb.Descriptors.ContentIdentifierDescriptor; namespace skyscraper8.GS.GSE_BFBS { @@ -200,6 +201,8 @@ namespace skyscraper8.GS.GSE_BFBS private void HandleAssembledFrame(ushort protocolType, byte[] buffer, GseLabel label) { + Context.UiJunction?.EnableUiFeature(SkyscraperUiFeature.GseAnalysis); + Context.UiJunction?.NotifyGsePacket(protocolType, buffer, label); switch (protocolType) { case 0x0081: diff --git a/skyscraper8/GS/GSE-HEM/GseHemReader.cs b/skyscraper8/GS/GSE-HEM/GseHemReader.cs index 028571f..f1dcbac 100644 --- a/skyscraper8/GS/GSE-HEM/GseHemReader.cs +++ b/skyscraper8/GS/GSE-HEM/GseHemReader.cs @@ -3,6 +3,8 @@ using skyscraper5.Dvb.DataBroadcasting; using skyscraper5.Skyscraper.IO; using skyscraper8.GS; using skyscraper8.GSE.GSE; +using skyscraper8.Skyscraper.Scraper; +using static skyscraper5.Dvb.Descriptors.ContentIdentifierDescriptor; namespace skyscraper8.GSE.GSE_HEM; @@ -98,7 +100,9 @@ class GseHemReader : IMisHandler private void HandlePacket(GsePacket packet) { - if (packet.StartIndicator && packet.EndIndicator) + Context.UiJunction?.EnableUiFeature(SkyscraperUiFeature.GseAnalysis); + Context.UiJunction?.NotifyGsePacket(packet.ProtocolType.Value, packet.GseDataBytes, packet.Label); + if (packet.StartIndicator && packet.EndIndicator) { switch (packet.ProtocolType) { diff --git a/skyscraper8/GS/GSE/GseLabel.cs b/skyscraper8/GS/GSE/GseLabel.cs index 7625c0d..cfe9141 100644 --- a/skyscraper8/GS/GSE/GseLabel.cs +++ b/skyscraper8/GS/GSE/GseLabel.cs @@ -9,6 +9,11 @@ public abstract class GseLabel public abstract int LabelTypeIndicator { get; } public abstract bool IsBroadcast(); + + public abstract int GetHashCode(); + public abstract bool Equals(object? obj); + + public abstract long ToLong(); } public class _6byteLabel : GseLabel @@ -38,6 +43,29 @@ public class _6byteLabel : GseLabel { return MAC.Equals(BROADCAST); } + + public override bool Equals(object? obj) + { + return obj is _6byteLabel label && + EqualityComparer.Default.Equals(MAC, label.MAC); + } + + public override int GetHashCode() + { + return HashCode.Combine(MAC); + } + + public override long ToLong() + { + byte[] bytes = MAC.GetAddressBytes(); + long result = 0; + for (int i = 0; i < bytes.Length; i++) + { + result = (result << 8); + result += bytes[i]; + } + return result; + } } public class _3byteLabel : GseLabel @@ -74,6 +102,30 @@ public class _3byteLabel : GseLabel { return buffer[1] == 0xff && buffer[2] == 0xff; } + + public override bool Equals(object? obj) + { + return obj is _3byteLabel label && + GroupId == label.GroupId && + LogonId == label.LogonId; + } + + public override int GetHashCode() + { + return HashCode.Combine(GroupId, LogonId); + } + + public override long ToLong() + { + long result = 0; + result = buffer[0]; + result <<= 8; + result = buffer[1]; + result <<= 8; + result = buffer[2]; + result <<= 8; + return result; + } } public class BroadcastLabel : GseLabel @@ -103,4 +155,19 @@ public class BroadcastLabel : GseLabel { return true; } + + public override bool Equals(object? obj) + { + return obj is BroadcastLabel label; + } + + public override int GetHashCode() + { + return 0; + } + + public override long ToLong() + { + return long.MaxValue; + } } \ No newline at end of file diff --git a/skyscraper8/GS/GSE/GseReader.cs b/skyscraper8/GS/GSE/GseReader.cs index 089bd07..8abf220 100644 --- a/skyscraper8/GS/GSE/GseReader.cs +++ b/skyscraper8/GS/GSE/GseReader.cs @@ -2,6 +2,7 @@ using log4net; using skyscraper5.Dvb.DataBroadcasting; using skyscraper5.Skyscraper.IO; using skyscraper8.GS; +using skyscraper8.Skyscraper.Scraper; namespace skyscraper8.GSE.GSE; @@ -152,6 +153,8 @@ internal class GseReader : IMisHandler throw new NotImplementedException(child.ToString()); } + Context.UiJunction?.EnableUiFeature(SkyscraperUiFeature.GseAnalysis); + Context.UiJunction?.NotifyGsePacket(child.ProtocolType.Value, child.GseDataBytes, child.Label); if (child.StartIndicator && child.EndIndicator) { switch (child.ProtocolType) diff --git a/skyscraper8/GS/GsContextDto.cs b/skyscraper8/GS/GsContextDto.cs index d543a60..3b40a0a 100644 --- a/skyscraper8/GS/GsContextDto.cs +++ b/skyscraper8/GS/GsContextDto.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Reflection; using System.Text; @@ -40,6 +41,7 @@ namespace skyscraper8.GS public IMultiprotocolEncapsulationEventHandler IpOutput { get; set; } public InteractionChannelHandler Rcs2Output { get; set; } + public ISkyscraperUiJunction UiJunction { get; internal set; } private Dictionary trafficTypes; diff --git a/skyscraper8/Mpeg2/TsAdaptionField.cs b/skyscraper8/Mpeg2/TsAdaptionField.cs index ab07cf4..8a274cc 100644 --- a/skyscraper8/Mpeg2/TsAdaptionField.cs +++ b/skyscraper8/Mpeg2/TsAdaptionField.cs @@ -16,6 +16,7 @@ namespace skyscraper5.Mpeg2 return; } byte[] adaptionField = binaryReader.ReadBytes(adaptionFieldLength); + RawAdaptionField = adaptionField; binaryReader = new MemoryStream(adaptionField); this.Length = adaptionFieldLength; @@ -178,5 +179,6 @@ namespace skyscraper5.Mpeg2 public bool PcrPresent { get; private set; } public byte Length { get; private set; } + public byte[] RawAdaptionField { get; private set; } } } diff --git a/skyscraper8/Program.cs b/skyscraper8/Program.cs index b1f9826..aae6c2f 100644 --- a/skyscraper8/Program.cs +++ b/skyscraper8/Program.cs @@ -24,6 +24,7 @@ using System.Net; using System.Net.NetworkInformation; using System.Net.Sockets; using System.Runtime.InteropServices; +using skyscraper5.Dvb.DataBroadcasting; using skyscraper8; using skyscraper8.GSE; using skyscraper8.Skyscraper.Math; @@ -111,6 +112,18 @@ namespace skyscraper5 return; } + if (args[0].ToLowerInvariant().Equals("bumutest")) + { + FileInfo fi = new FileInfo(args[1]); + FileStream fstream = fi.OpenRead(); + MultiprotocolEncapsulationDecoder mpeDecoder = new MultiprotocolEncapsulationDecoder(null); + PsiDecoder psiDecoder = new PsiDecoder(0x1019, mpeDecoder); + TsContext mpeg2 = new TsContext(); + mpeg2.RegisterPacketProcessor(0x1019, psiDecoder); + SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, new InMemoryScraperStorage(), new NullObjectStorage()); + skyscraper.InitalizeFilterChain(); + skyscraper.IngestFromStream(fstream); + } if (args[0].Equals("aactest")) { new AacTestProgram().Run(); diff --git a/skyscraper8/Properties/launchSettings.json b/skyscraper8/Properties/launchSettings.json index d59f0c7..a02e1d2 100644 --- a/skyscraper8/Properties/launchSettings.json +++ b/skyscraper8/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "skyscraper8": { "commandName": "Project", - "commandLineArgs": "cscan tcp://127.0.0.1:6969", + "commandLineArgs": "bumutest \"C:\\devel\\skyscraper8-testsuite\\12168_pid1019only.ts\"", "remoteDebugEnabled": false }, "Container (Dockerfile)": { diff --git a/skyscraper8/Ses/SgtService.cs b/skyscraper8/Ses/SgtService.cs index cd779cd..ade59d5 100644 --- a/skyscraper8/Ses/SgtService.cs +++ b/skyscraper8/Ses/SgtService.cs @@ -47,7 +47,7 @@ namespace skyscraper8.Ses return HashCode.Combine(ServiceListId, ServiceId, TransportStreamId, OriginalNetworkId); } - internal string GetName() + public string GetName() { if (ServiceDescriptor == null) return "???"; diff --git a/skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs b/skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs index 4ccb27d..69b38e0 100644 --- a/skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs +++ b/skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs @@ -89,5 +89,10 @@ namespace skyscraper8.Skyscraper.Math return (Entropy / 8.0) * 100.0; } } + + public void Cheat(double newValue) + { + _entropy = newValue; + } } } diff --git a/skyscraper8/Skyscraper/Net/IpTrafficInfo.cs b/skyscraper8/Skyscraper/Net/IpTrafficInfo.cs index 31a83c1..bfe0f78 100644 --- a/skyscraper8/Skyscraper/Net/IpTrafficInfo.cs +++ b/skyscraper8/Skyscraper/Net/IpTrafficInfo.cs @@ -104,5 +104,15 @@ namespace skyscraper5.Skyscraper.Net sb.Append(")"); return sb.ToString(); } + + public string RenderSource() + { + return string.IsNullOrEmpty(SourceName) ? Source.ToString() : SourceName; + } + + public string RenderTarget() + { + return string.IsNullOrEmpty(TargetName) ? Target.ToString() : TargetName; + } } } diff --git a/skyscraper8/Skyscraper/Scraper/ISkyscraperUiJunction.cs b/skyscraper8/Skyscraper/Scraper/ISkyscraperUiJunction.cs index 7760835..32de9b4 100644 --- a/skyscraper8/Skyscraper/Scraper/ISkyscraperUiJunction.cs +++ b/skyscraper8/Skyscraper/Scraper/ISkyscraperUiJunction.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; +using moe.yo3explorer.skyscraper8.DVBI.Model; using skyscraper5.Docsis; using skyscraper5.Docsis.MacManagement; using skyscraper5.Dvb.DataBroadcasting.IntModel; @@ -18,7 +19,12 @@ using skyscraper5.Skyscraper.IO.CrazycatStreamReader; using skyscraper5.Skyscraper.Net; using skyscraper5.src.Skyscraper.FrequencyListGenerator; using skyscraper5.Teletext.Wss; +using skyscraper8.DvbNip; +using skyscraper8.DvbNip.UiIntegration; using skyscraper8.GSE; +using skyscraper8.GSE.GSE; +using skyscraper8.Ietf.FLUTE; +using skyscraper8.Ses; using skyscraper8.Skyscraper.Drawing; using skyscraper8.Skyscraper.FrequencyListGenerator; using skyscraper8.Skyscraper.Scraper; @@ -36,7 +42,7 @@ namespace skyscraper5.Skyscraper.Scraper void DsmCcModuleAdd(int elementaryPid, ushort moduleInfoModuleId, byte moduleInfoModuleVersion); void DsmCcModuleProgress(int elementaryPid, ushort moduleInfoModuleId, byte moduleInfoModuleVersion, double moduleInfoDownloadProgress); void DsmCcModuleComplete(int elementaryPid, ushort moduleModuleId, byte moduleModuleVersion); - void NotifyWss(ushort programNumber, WssDataBlock wssDataBlock); + void NotifyWss(ushort programNumber, WssDataBlock wssDataBlock, int pid); void NotifyStreamTypeDetection(string contestantTag, int pid); void NotifyBat(BatBouquet batBouquet); void DsmCcVfs(VfsFile vfsFile); @@ -198,7 +204,21 @@ namespace skyscraper5.Skyscraper.Scraper /// The payload begins at byte 11 void OnBbframe(BBHeader bbHeader, byte[] payload); void OnDetectionOfInnerTs(SkyscraperContext child, object identifier); + void NotifyGsePacket(ushort value, byte[] gseDataBytes, GseLabel label); + void OnDvbNipFileArrival(FluteUiHandle fuh); + void OnDvbNipMulticastGatewayConfiguration(NipActualCarrierInformation carrier, MulticastGatewayConfigurationType multicastGatewayConfiguration); + void OnDvbNipCarrierDetected(NipActualCarrierInformation currentCarrierInformation); + void OnDvbNipPrivateDataSignallingManifest(PrivateDataSignallingManifestType privateDataSignallingManifest); + void OnDvbNipServiceListEntryPoints(NipActualCarrierInformation currentCarrierInformation, ServiceListEntryPoints serviceListEntryPoints, DateTime dvbNipTime); + void OnDvbNipServiceList(NipActualCarrierInformation currentCarrierInformation, string serviceListId1, string serviceListId2); + void OnDvbNipTimeOffsetFile(NipActualCarrierInformation currentCarrierInformation, TimeOffsetFileType timeOffsetFile); + void OnDvbNipNetworkInformationFile(NipActualCarrierInformation currentCarrierInformation, NetworkInformationFileType networkInformationFile); + void DvbNipServiceInformation(NipActualCarrierInformation currentCarrierInformation, ServiceInformationFileType serviceInformationFile); + void OnDvbNipFileAnnouncement(FDTInstanceType flute); + void OnAstraSgtList(SgtList list); + void OnAstraSgtService(SgtService child); TaskQueue Tasks { get; set; } + } } diff --git a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs index 872bb70..bcec7f6 100644 --- a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs +++ b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs @@ -1,7 +1,10 @@ -using log4net; +using Ionic.Zlib; +using log4net; +using MimeKit; using moe.yo3explorer.skyscraper8.DVBI.Model; using skyscraper5.Abertis; using skyscraper5.Docsis; +using skyscraper5.Docsis.AnnexC; using skyscraper5.Docsis.MacManagement; using skyscraper5.DsmCc.Descriptors; using skyscraper5.Dvb.DataBroadcasting; @@ -43,6 +46,7 @@ using skyscraper5.src.Id3; using skyscraper5.src.InteractionChannel; using skyscraper5.src.InteractionChannel.Model; using skyscraper5.src.InteractionChannel.Model.Descriptors; +using skyscraper5.src.InteractionChannel.Model2; using skyscraper5.src.Mpeg2; using skyscraper5.src.Mpeg2.PacketFilter; using skyscraper5.src.Skyscraper.Scraper.Dns; @@ -53,40 +57,39 @@ using skyscraper5.T2MI.Packets.AdressingFunctions; using skyscraper5.Teletext; using skyscraper5.Teletext.Vps; using skyscraper5.Teletext.Wss; +using skyscraper8.Abertis; using skyscraper8.DvbI; using skyscraper8.DvbNip; -using skyscraper8.Ietf.FLUTE; -using skyscraper8.Ietf.Rfc4236_ULE; -using skyscraper8.Ses; -using skyscraper8.Skyscraper.Scraper.Storage; -using skyscraper8.yo3explorer; -using System.Collections.ObjectModel; -using System.Diagnostics; -using System.Net; -using System.Net.NetworkInformation; -using System.Reflection; -using System.Text; -using MimeKit; -using skyscraper5.src.InteractionChannel.Model2; -using skyscraper8.Abertis; +using skyscraper8.DvbNip.UiIntegration; +using skyscraper8.DvbSis; using skyscraper8.Experimentals.NdsSsu; using skyscraper8.Experimentals.OtvSsu; using skyscraper8.GS; using skyscraper8.GSE; using skyscraper8.Ieee802_1AB; +using skyscraper8.Ietf.FLUTE; +using skyscraper8.Ietf.Rfc4236_ULE; using skyscraper8.InteractionChannel.Model; using skyscraper8.InteractionChannel.Model2; using skyscraper8.InteractionChannel.Model2.Descriptors; +using skyscraper8.Ses; using skyscraper8.Skyscraper.Net; using skyscraper8.Skyscraper.Scraper; +using skyscraper8.Skyscraper.Scraper.Storage; using skyscraper8.T2MI; +using skyscraper8.T2MI.Packets; +using skyscraper8.yo3explorer; +using System.Collections.ObjectModel; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Net; +using System.Net.NetworkInformation; +using System.Reflection; +using System.Text; +using System.Threading; using Tsubasa.IO; using Platform = skyscraper5.Dvb.SystemSoftwareUpdate.Model.Platform; using RntParser = skyscraper5.Dvb.TvAnytime.RntParser; -using skyscraper8.DvbSis; -using skyscraper8.T2MI.Packets; -using skyscraper5.Docsis.AnnexC; -using Ionic.Zlib; namespace skyscraper5.Skyscraper.Scraper { @@ -254,13 +257,19 @@ namespace skyscraper5.Skyscraper.Scraper public void IngestFromStream(Stream stream, bool closeWhenDone = true) { byte[] buffer = new byte[188]; + int read = 0; while (true) { if (IsAbortConditionMet()) break; - if (stream.Read(buffer, 0, 188) != 188) + read = stream.Read(buffer, 0, 188); + + if (read != 188) + { + logger.Warn(String.Format("Reading {0} bytes failed. Got {1} instead.", 188, read)); break; + } try { @@ -341,18 +350,31 @@ namespace skyscraper5.Skyscraper.Scraper PropertyInfo[] properties = typeof(GsContextDto).GetProperties(); foreach (PropertyInfo propertyInfo in properties) { - object? value = propertyInfo.GetValue(child); - if (value == null) - { - throw new SkyscraperException(String.Format("While creating the {0}, the {1} was not properly set. This is a bug, tell Fey.", nameof(GsContextDto), propertyInfo.Name)); - } + if (!propertyInfo.Name.Equals(nameof(child.UiJunction))) + { + object? value = propertyInfo.GetValue(child); + if (value == null) + { + throw new SkyscraperException(String.Format("While creating the {0}, the {1} was not properly set. This is a bug, tell Fey.", nameof(GsContextDto), propertyInfo.Name)); + } + } } //actually return the object return child; } - private DocsisPacketProcessor docsisPacketProcessor; + private bool TestNullAllowed(PropertyInfo prop) + { + //AllowNull + Attribute? attribute = prop.GetCustomAttribute(typeof(AllowNullAttribute)); + if (attribute != null) + return true; + else + return false; + } + + private DocsisPacketProcessor docsisPacketProcessor; public int SpecialTsType { get; private set; } private void CheckSpecialTs() { @@ -600,7 +622,7 @@ namespace skyscraper5.Skyscraper.Scraper break; if (!CurrentTransportStreamId.HasValue) break; - TeletextPesProcessor ttp = new TeletextPesProcessor(this, CurrentNetworkId.Value, CurrentTransportStreamId.Value, result.ProgramNumber); + TeletextPesProcessor ttp = new TeletextPesProcessor(this, CurrentNetworkId.Value, CurrentTransportStreamId.Value, result.ProgramNumber, mappingStream.ElementaryPid); if (result.PrivateDataSpecifier.HasValue) ttp.PrivateDataSpecifier = result.PrivateDataSpecifier; if (mappingStream.PrivateDataSpecifier.HasValue) @@ -1169,7 +1191,7 @@ namespace skyscraper5.Skyscraper.Scraper private HashSet _vpsCoordinates; - public void OnVpsData(int networkId, int transportStreamId, ushort programNumber, VpsDataBlock vpsDataField) + public void OnVpsData(int networkId, int transportStreamId, ushort programNumber, VpsDataBlock vpsDataField, int elementaryPid) { //We don't really need VPS data, but we'll use it as another way of making sure we //got everything. @@ -1191,7 +1213,7 @@ namespace skyscraper5.Skyscraper.Scraper private Dictionary _wssDataBlocks; - public void OnWssData(int networkId, int transportStreamId, ushort programNumber, WssDataBlock wssDataBlock) + public void OnWssData(int networkId, int transportStreamId, ushort programNumber, WssDataBlock wssDataBlock, int elementaryPid) { if (_wssDataBlocks == null) { @@ -1202,7 +1224,7 @@ namespace skyscraper5.Skyscraper.Scraper { _wssDataBlocks.Add(programNumber, wssDataBlock); LogEvent(SkyscraperContextEvent.WssData, programNumber.ToString()); - UiJunction?.NotifyWss(programNumber, wssDataBlock); + UiJunction?.NotifyWss(programNumber, wssDataBlock, elementaryPid); } } @@ -2727,6 +2749,8 @@ namespace skyscraper5.Skyscraper.Scraper void SgtEventHandler.AnnounceSgtList(SgtList list) { + UiJunction?.OnAstraSgtList(list); + if (!DataStorage.TestForSgtList(list)) { LogEvent(SkyscraperContextEvent.SgtList, String.Format("List #{0} ({1})", list.ServiceListId, list.GetName())); @@ -2741,6 +2765,7 @@ namespace skyscraper5.Skyscraper.Scraper void SgtEventHandler.OnSgtService(SgtService child) { + UiJunction?.OnAstraSgtService(child); if (!DataStorage.TestForSgtService(child)) { LogEvent(SkyscraperContextEvent.SgtService, String.Format("LCN #{0} in List #{1} ({2})", child.Lcn, child.ServiceListId, child.GetName())); @@ -2773,6 +2798,8 @@ namespace skyscraper5.Skyscraper.Scraper public void FluteFileArrival(NipActualCarrierInformation carrier, FluteListener listener) { + + Stream stream = listener.ToStream(); bool isMime = DvbNipUtilities.IsMime(stream); if (isMime) @@ -2822,6 +2849,8 @@ namespace skyscraper5.Skyscraper.Scraper public void OnMulticastGatewayConfiguration(NipActualCarrierInformation carrier, MulticastGatewayConfigurationType multicastGatewayConfiguration) { + UiJunction?.OnDvbNipMulticastGatewayConfiguration(carrier, multicastGatewayConfiguration); + if (multicastGatewayConfiguration.MulticastSession != null) { foreach (MulticastSessionType multicastSession in multicastGatewayConfiguration.MulticastSession) @@ -2861,6 +2890,8 @@ namespace skyscraper5.Skyscraper.Scraper public void OnNipCarrierDetected(NipActualCarrierInformation currentCarrierInformation) { + UiJunction?.EnableUiFeature(SkyscraperUiFeature.DvbNipAnalyis); + UiJunction?.OnDvbNipCarrierDetected(currentCarrierInformation); LogEvent(SkyscraperContextEvent.NipCarrierDetected, String.Format("{0}", currentCarrierInformation.NipStreamProviderName)); if (!DataStorage.DvbNipTestForCarrier(currentCarrierInformation)) { @@ -2878,6 +2909,7 @@ namespace skyscraper5.Skyscraper.Scraper public void OnPrivateDataSignallingManifest(NipActualCarrierInformation currentCarrierInformation, PrivateDataSignallingManifestType privateDataSignallingManifest) { + UiJunction?.OnDvbNipPrivateDataSignallingManifest(privateDataSignallingManifest); foreach (PrivateDataProviderType privateDataProvider in privateDataSignallingManifest.PrivateDataProvider) { privateDataProvider.privateDataProviderID.FlipEndian(); @@ -2893,6 +2925,8 @@ namespace skyscraper5.Skyscraper.Scraper public void OnServiceListEntryPoints(NipActualCarrierInformation currentCarrierInformation, ServiceListEntryPoints serviceListEntryPoints, DateTime dvbNipTime, DvbNipServiceListNotifier serviceListNotifier) { + UiJunction?.OnDvbNipServiceListEntryPoints(currentCarrierInformation, serviceListEntryPoints, dvbNipTime); + long sourceHash = DvbIUtils.GetSourceHash(ServiceListEntryPointSource.DvbNip, currentCarrierInformation.NipNetworkId, currentCarrierInformation.NipCarrierId, currentCarrierInformation.NipLinkId); serviceListNotifier.SetSourceHash(sourceHash); DvbIDataStorage dataStorage = DataStorage; @@ -2907,7 +2941,7 @@ namespace skyscraper5.Skyscraper.Scraper else { logger.InfoFormat("New DVB-I Service List Entry Point: {0}", currentCarrierInformation.NipStreamProviderName); - dataStorage.InsertDvbiServiceListEntryPoint(sourceHash); + dataStorage.InsertDvbiServiceListEntryPoint(sourceHash); } IEnumerable enumerable = DvbIUtils.FlattenServiceListEntryPoints(serviceListEntryPoints); @@ -2946,6 +2980,8 @@ namespace skyscraper5.Skyscraper.Scraper public void OnServiceList(NipActualCarrierInformation currentCarrierInformation, string serviceListId, ServiceListType serviceList) { + UiJunction?.OnDvbNipServiceList(currentCarrierInformation, serviceListId, serviceListId); + List services = DvbIUtils.FlattenServiceList(serviceList).ToList(); DvbIDataStorage dataStorage = DataStorage; foreach (DvbIService service in services) @@ -2969,6 +3005,7 @@ namespace skyscraper5.Skyscraper.Scraper public void OnTimeOffsetFile(NipActualCarrierInformation currentCarrierInformation, TimeOffsetFileType timeOffsetFile) { + UiJunction?.OnDvbNipTimeOffsetFile(currentCarrierInformation, timeOffsetFile); if (timeOffsetFile.time_of_change != DateTime.MinValue) { throw new NotImplementedException(); @@ -2977,6 +3014,7 @@ namespace skyscraper5.Skyscraper.Scraper public void OnNetworkInformationFile(NipActualCarrierInformation currentCarrierInformation, NetworkInformationFileType networkInformationFile) { + UiJunction?.OnDvbNipNetworkInformationFile(currentCarrierInformation, networkInformationFile); List broadcastNetworks = new List(); if (networkInformationFile.ActualBroadcastNetwork != null) broadcastNetworks.Add(networkInformationFile.ActualBroadcastNetwork); @@ -2995,6 +3033,7 @@ namespace skyscraper5.Skyscraper.Scraper public void OnServiceInformationFile(NipActualCarrierInformation currentCarrierInformation, ServiceInformationFileType serviceInformationFile) { + UiJunction?.DvbNipServiceInformation(currentCarrierInformation, serviceInformationFile); foreach (BroadcastMediaStreamType broadcastMediaStreamType in serviceInformationFile.BroadcastMediaStream) { if (!DataStorage.DvbNipTestForService(broadcastMediaStreamType)) @@ -3012,6 +3051,8 @@ namespace skyscraper5.Skyscraper.Scraper private HashSet knownUrls; public void FluteFileAnnouncement(IPAddress ip, ushort port, FDTInstanceType flute) { + UiJunction?.OnDvbNipFileAnnouncement(flute); + if (knownUrls == null) { knownUrls = new HashSet(); diff --git a/skyscraper8/Skyscraper/Scraper/SkyscraperUiFeature.cs b/skyscraper8/Skyscraper/Scraper/SkyscraperUiFeature.cs index a633791..6731d45 100644 --- a/skyscraper8/Skyscraper/Scraper/SkyscraperUiFeature.cs +++ b/skyscraper8/Skyscraper/Scraper/SkyscraperUiFeature.cs @@ -11,6 +11,8 @@ namespace skyscraper8.Skyscraper.Scraper BbframeAnalysis, DocsisAnalysis, BlockstreamAnalysis, - IpTrafficAnalysis + IpTrafficAnalysis, + GseAnalysis, + DvbNipAnalyis } } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/Utilities/DatabaseKeyNipService.cs b/skyscraper8/Skyscraper/Scraper/Storage/Utilities/DatabaseKeyNipService.cs new file mode 100644 index 0000000..945fb8e --- /dev/null +++ b/skyscraper8/Skyscraper/Scraper/Storage/Utilities/DatabaseKeyNipService.cs @@ -0,0 +1,39 @@ +using skyscraper8.DvbNip; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.Skyscraper.Scraper.Storage.Utilities +{ + public class DatabaseKeyNipService + { + public DatabaseKeyNipService(BroadcastMediaStreamType mediaStreamType) + { + this.LinkId = mediaStreamType.NIPLinkID; + this.CarrierId = mediaStreamType.NIPCarrierID; + this.NetworkId = mediaStreamType.NIPNetworkID; + this.ServiceId = mediaStreamType.NIPServiceID; + } + + public string LinkId { get; } + public string CarrierId { get; } + public ushort NetworkId { get; } + public ushort ServiceId { get; } + + public override bool Equals(object? obj) + { + return obj is DatabaseKeyNipService service && + LinkId == service.LinkId && + CarrierId == service.CarrierId && + NetworkId == service.NetworkId && + ServiceId == service.ServiceId; + } + + public override int GetHashCode() + { + return HashCode.Combine(LinkId, CarrierId, NetworkId, ServiceId); + } + } +} diff --git a/skyscraper8/Skyscraper/Scraper/Storage/Utilities/NipPrivateDataSpecifierDto.cs b/skyscraper8/Skyscraper/Scraper/Storage/Utilities/NipPrivateDataSpecifierDto.cs new file mode 100644 index 0000000..4838173 --- /dev/null +++ b/skyscraper8/Skyscraper/Scraper/Storage/Utilities/NipPrivateDataSpecifierDto.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.Skyscraper.Scraper.Storage.Utilities +{ + public class NipPrivateDataSpecifierDto + { + public DateTime VersionDate { get; set; } + } +} diff --git a/skyscraper8/Skyscraper/Scraper/StreamAutodetection/Contestants/TeletextContestant.cs b/skyscraper8/Skyscraper/Scraper/StreamAutodetection/Contestants/TeletextContestant.cs index 0201a57..ab4e14d 100644 --- a/skyscraper8/Skyscraper/Scraper/StreamAutodetection/Contestants/TeletextContestant.cs +++ b/skyscraper8/Skyscraper/Scraper/StreamAutodetection/Contestants/TeletextContestant.cs @@ -19,15 +19,15 @@ namespace skyscraper5.Skyscraper.Scraper.StreamAutodetection.Contestants { public TeletextContestant(int pid) : base("Teletext", pid) { - PacketProcessor = new PesDecoder(new TeletextPesProcessor(this, 0, 0, 0)); + PacketProcessor = new PesDecoder(new TeletextPesProcessor(this, 0, 0, 0, pid)); } - public void OnVpsData(int networkId, int transportStreamId, ushort programNumber, VpsDataBlock vpsDataField) + public void OnVpsData(int networkId, int transportStreamId, ushort programNumber, VpsDataBlock vpsDataField, int elementaryPid) { Score++; } - public void OnWssData(int networkId, int transportStreamId, ushort programNumber, WssDataBlock wssDataBlock) + public void OnWssData(int networkId, int transportStreamId, ushort programNumber, WssDataBlock wssDataBlock, int elementaryPid) { Score++; } @@ -62,7 +62,7 @@ namespace skyscraper5.Skyscraper.Scraper.StreamAutodetection.Contestants { programContext.Program = new ProgramMapping(0x1fff, 0x1fff); } - TeletextPesProcessor ttp = new TeletextPesProcessor(skyscraperContext, skyscraperContext.CurrentNetworkId.Value, skyscraperContext.CurrentTransportStreamId.Value, programContext.ProgramNumber); + TeletextPesProcessor ttp = new TeletextPesProcessor(skyscraperContext, skyscraperContext.CurrentNetworkId.Value, skyscraperContext.CurrentTransportStreamId.Value, programContext.ProgramNumber, pid); ttp.PrivateDataSpecifier = programContext.PrivateDataSpecifier; skyscraperContext.DvbContext.RegisterPacketProcessor(pid, new PesDecoder(ttp)); } diff --git a/skyscraper8/Skyscraper/Text/SkyscraperBaseEncoding8.cs b/skyscraper8/Skyscraper/Text/SkyscraperBaseEncoding8.cs index e79aeb8..d99e202 100644 --- a/skyscraper8/Skyscraper/Text/SkyscraperBaseEncoding8.cs +++ b/skyscraper8/Skyscraper/Text/SkyscraperBaseEncoding8.cs @@ -117,24 +117,31 @@ namespace skyscraper5.Skyscraper.Text return lastDecrypted.Length; } + private object getCharsLocker; public override int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex) { - if (!IsSameStringAsLast(bytes, byteIndex, byteCount)) - { - int preProcessLength = GetPreProcessLength(bytes, byteIndex, byteCount); - byte[] preProcess = PreProcess(bytes, byteIndex, byteCount, preProcessLength); - int maxAllowedSize = lastDecrypted.Length; - char[] overwriting = Decrypt(preProcess); - if (overwriting.Length > maxAllowedSize) - { - throw new AccessViolationException("Oh no, I guess the multithreading went haywire..."); - } - lastDecrypted = overwriting; - } + if (getCharsLocker == null) + getCharsLocker = new object(); - int result = Math.Min(chars.Length - charIndex, lastDecrypted.Length); - Array.Copy(lastDecrypted, 0, chars, charIndex, result); - return result; + lock (getCharsLocker) + { + if (!IsSameStringAsLast(bytes, byteIndex, byteCount)) + { + int preProcessLength = GetPreProcessLength(bytes, byteIndex, byteCount); + byte[] preProcess = PreProcess(bytes, byteIndex, byteCount, preProcessLength); + int maxAllowedSize = lastDecrypted.Length; + char[] overwriting = Decrypt(preProcess); + if (overwriting.Length > maxAllowedSize) + { + throw new AccessViolationException("Oh no, I guess the multithreading went haywire..."); + } + lastDecrypted = overwriting; + } + + int result = Math.Min(chars.Length - charIndex, lastDecrypted.Length); + Array.Copy(lastDecrypted, 0, chars, charIndex, result); + return result; + } } public override int GetMaxByteCount(int charCount) diff --git a/skyscraper8/Teletext/ITeletextPageHandler.cs b/skyscraper8/Teletext/ITeletextPageHandler.cs index 599ccb9..95f9008 100644 --- a/skyscraper8/Teletext/ITeletextPageHandler.cs +++ b/skyscraper8/Teletext/ITeletextPageHandler.cs @@ -11,8 +11,8 @@ namespace skyscraper5.Teletext { interface ITeletextPageHandler { - void OnVpsData(int networkId, int transportStreamId, ushort programNumber, VpsDataBlock vpsDataField); - void OnWssData(int networkId, int transportStreamId, ushort programNumber, WssDataBlock wssDataBlock); + void OnVpsData(int networkId, int transportStreamId, ushort programNumber, VpsDataBlock vpsDataField, int elementaryPid); + void OnWssData(int networkId, int transportStreamId, ushort programNumber, WssDataBlock wssDataBlock, int elementaryPid); void OnTeletextPage(int networkId, int transportStreamId, ushort programNumber, TeletextMagazine magazine); void OnMonochromeData(int networkId, int transportStreamId, ushort programNumber, MonochromeDataField result); } diff --git a/skyscraper8/Teletext/TeletextPesProcessor.cs b/skyscraper8/Teletext/TeletextPesProcessor.cs index 7dfcad0..44019e1 100644 --- a/skyscraper8/Teletext/TeletextPesProcessor.cs +++ b/skyscraper8/Teletext/TeletextPesProcessor.cs @@ -19,14 +19,16 @@ namespace skyscraper5.Teletext public int NetworkId { get; } public int TransportStreamId { get; } public ushort ProgramNumber { get; } + public int ElementaryPid { get; } - public TeletextPesProcessor(ITeletextPageHandler pageHandler, int networkId, int transportStreamId, ushort programNumber) + public TeletextPesProcessor(ITeletextPageHandler pageHandler, int networkId, int transportStreamId, ushort programNumber, int elementaryPid) { PageHandler = pageHandler; NetworkId = networkId; TransportStreamId = transportStreamId; ProgramNumber = programNumber; - Magazines = new TeletextMagazine[8]; + ElementaryPid = elementaryPid; + Magazines = new TeletextMagazine[8]; } public TeletextMagazine[] Magazines { get; private set; } @@ -124,13 +126,13 @@ namespace skyscraper5.Teletext private void Wss(byte[] buffer) { WssDataField wssDataField = new WssDataField(buffer); - PageHandler.OnWssData(NetworkId, TransportStreamId, ProgramNumber, wssDataField.WssDataBlock); + PageHandler.OnWssData(NetworkId, TransportStreamId, ProgramNumber, wssDataField.WssDataBlock, ElementaryPid); } private void Vps(byte[] buffer) { VpsDataField vpsDataField = new VpsDataField(buffer); - PageHandler.OnVpsData(NetworkId, TransportStreamId, ProgramNumber, vpsDataField.VpsDataBlock); + PageHandler.OnVpsData(NetworkId, TransportStreamId, ProgramNumber, vpsDataField.VpsDataBlock, ElementaryPid); } private void EbuTeletext(byte[] buffer) diff --git a/skyscraper8/skyscraper8.csproj b/skyscraper8/skyscraper8.csproj index 45804f8..18aa551 100644 --- a/skyscraper8/skyscraper8.csproj +++ b/skyscraper8/skyscraper8.csproj @@ -24,9 +24,9 @@ - - - + + +