Patches for Voile

skyscraper8/GS/BBframeDeencapsulator3.cs

@@ -32,6 +32,7 @@ class BbframeDeencapsulator3 : IBbframeDeencapsulator
         if (!bbHeader.Valid)
             return;
 
+
         if (mis == null)
             mis = new IMisHandler[256];
         if (mis[bbHeader.Matype2] == null)
@@ -40,6 +41,7 @@ class BbframeDeencapsulator3 : IBbframeDeencapsulator
         }
 
 		mis[bbHeader.Matype2].PushFrame(bbHeader, new ReadOnlySpan<byte>(bbframe, 11, bbframe.Length - 11));
+		context.UiJunction.OnBbframe(bbHeader, bbframe);
 	}
 
     private IMisHandler[] mis;

skyscraper8/GS/GsContextDto.cs

@@ -5,6 +5,7 @@ using System.Reflection;
 using System.Text;
 using System.Threading.Tasks;
 using skyscraper5.Dvb.DataBroadcasting;
+using skyscraper5.Skyscraper.Scraper;
 using skyscraper5.src.InteractionChannel;
 using skyscraper8.Skyscraper.Scraper;
 
@@ -39,6 +40,7 @@ namespace skyscraper8.GS
 		public IMultiprotocolEncapsulationEventHandler IpOutput { get; set; }
 
 		public InteractionChannelHandler Rcs2Output { get; set; }
+		public ISkyscraperUiJunction UiJunction { get; internal set; }
 
 		private Dictionary<ushort, ulong> trafficTypes;
 		public void InformTrafficType(ushort trafficType)

skyscraper8/Mpeg2/Psi/Model/ProgramMappingStream.cs

@@ -208,18 +208,22 @@ namespace skyscraper5.Mpeg2.Psi.Model
 		public byte[] AudioSpecifConfig { get; internal set; }
 		public byte[] AudioProfileLevelIndication { get; internal set; }
 		public uint? MetadataApplicationFormatIdentifier { get; internal set; }
-		public ushort MetadataApplicationFormat { get; internal set; }
+		public ushort? MetadataApplicationFormat { get; internal set; }
-		public byte MetadataFormat { get; internal set; }
+		public byte? MetadataFormat { get; internal set; }
 		public uint? MetadataFormatIdentifier { get; internal set; }
-		public byte MetadataServiceId { get; internal set; }
+		public byte? MetadataServiceId { get; internal set; }
-		public int DecoderConfigFlag { get; internal set; }
+		public int? DecoderConfigFlag { get; internal set; }
 		public byte[] ServiceIdentificationRecord { get; internal set; }
 		public byte[] DecoderConfig { get; internal set; }
 		public byte[] DecoderConfigIdentificationRecord { get; internal set; }
-		public ushort DecoderConfigMetadataServiceId { get; internal set; }
+		public ushort? DecoderConfigMetadataServiceId { get; internal set; }
 		public byte[] Reserved { get; internal set; }
 		public byte[] PrivateData { get; internal set; }
 
+        //Voile tools
+		public bool Ac3DescriptorPresent { get; internal set; }
+		public int NumDescriptors { get; internal set; }
+
 		public List<UserDefinedDescriptor> UnknownUserDefines;
 		public ProgramMappingStream(PmtStreamType streamType, int elementaryPid)
         {

skyscraper8/Mpeg2/Psi/PmtParser.cs

@@ -65,7 +65,11 @@ namespace skyscraper5.Mpeg2.Psi
             ProgramMapping result = new ProgramMapping(programNumber, pcrPid);
             if (programInfoLength > 0)
             {
-                byte[] descriptorBuffer = buffer.ReadBytes(programInfoLength);
+                byte[] descriptorBuffer = new byte[programInfoLength];
+                int descriptorBufferFilling = buffer.Read(descriptorBuffer, 0, programInfoLength);
+                if (descriptorBufferFilling != programInfoLength)
+                    return;
+
                 ParseOuterDescriptors(result, descriptorBuffer);
             }
 
@@ -97,6 +101,7 @@ namespace skyscraper5.Mpeg2.Psi
             IEnumerable<TsDescriptor> descriptors = TsDescriptorUnpacker.GetInstance().UnpackDescriptors(inputBuffer, "PMT");
             foreach (TsDescriptor dvbDescriptor in descriptors)
             {
+                output.NumDescriptors++;
                 switch (dvbDescriptor.GetType().Name)
                 {
                     case nameof(StreamIdentifierDescriptor):
@@ -140,6 +145,7 @@ namespace skyscraper5.Mpeg2.Psi
                         output.BSID = ac3Descriptor.BSID;
                         output.ComponentType = ac3Descriptor.ComponentType;
                         output.MainId = ac3Descriptor.MainId;
+                        output.Ac3DescriptorPresent = true;
                         break;
                     case nameof(VideoStreamDescriptor):
                         VideoStreamDescriptor videoStreamDescriptor = (VideoStreamDescriptor)dvbDescriptor;

skyscraper8/Skyscraper/IO/StreamExtensions.cs

@@ -11,11 +11,10 @@ namespace skyscraper5.Skyscraper.IO
 {
     public static class StreamExtensions
     {
-        private static byte[] buffer = new byte[8];
-
         [DebuggerStepThrough]
         public static ushort ReadUInt16BE(this Stream stream)
         {
+			byte[] buffer = new byte[2];
             if (stream.Read(buffer, 0, 2) != 2)
                 throw new EndOfStreamException();
             if (BitConverter.IsLittleEndian)
@@ -25,6 +24,7 @@ namespace skyscraper5.Skyscraper.IO
 
 		public static short ReadInt16LE(this Stream stream)
 		{
+			byte[] buffer = new byte[2];
 			if (stream.Read(buffer, 0, 2) != 2)
 				throw new EndOfStreamException();
 			if (!BitConverter.IsLittleEndian)
@@ -34,6 +34,7 @@ namespace skyscraper5.Skyscraper.IO
 
 		public static ushort ReadUInt16LE(this Stream stream)
         {
+			byte[] buffer = new byte[2];
 			if (stream.Read(buffer, 0, 2) != 2)
                 throw new EndOfStreamException();
             if (!BitConverter.IsLittleEndian)
@@ -55,6 +56,7 @@ namespace skyscraper5.Skyscraper.IO
 		[DebuggerStepThrough]
 		public static uint ReadUInt32LE(this Stream stream)
 		{
+			byte[] buffer = new byte[4];
 			if (stream.Read(buffer, 0, 4) != 4)
 				throw new EndOfStreamException();
 			if (!BitConverter.IsLittleEndian)
@@ -64,6 +66,7 @@ namespace skyscraper5.Skyscraper.IO
 
 		public static int ReadInt32LE(this Stream stream)
         {
+			byte[] buffer = new byte[4];
 			if (stream.Read(buffer, 0, 4) != 4)
                 throw new EndOfStreamException();
             if (!BitConverter.IsLittleEndian)
@@ -82,6 +85,7 @@ namespace skyscraper5.Skyscraper.IO
 
 		public static long ReadInt64LE(this Stream stream)
 		{
+			byte[] buffer = new byte[8];
 			if (stream.Read(buffer, 0, 8) != 8)
 				throw new EndOfStreamException();
 			if (!BitConverter.IsLittleEndian)
@@ -91,6 +95,7 @@ namespace skyscraper5.Skyscraper.IO
 
 		public static ulong ReadUInt64LE(this Stream stream)
 		{
+			byte[] buffer = new byte[8];
 			if (stream.Read(buffer, 0, 8) != 8)
 				throw new EndOfStreamException();
 			if (!BitConverter.IsLittleEndian)
@@ -100,6 +105,7 @@ namespace skyscraper5.Skyscraper.IO
 
 		public static ulong ReadUInt64BE(this Stream stream)
         {
+			byte[] buffer = new byte[8];
 			if (stream.Read(buffer, 0, 8) != 8)
                 throw new EndOfStreamException();
             if (BitConverter.IsLittleEndian)
@@ -150,6 +156,7 @@ namespace skyscraper5.Skyscraper.IO
         [DebuggerStepThrough]
         public static byte ReadUInt8(this Stream stream)
         {
+			byte[] buffer = new byte[1];
 			if (stream.Read(buffer, 0, 1) != 1)
                 throw new EndOfStreamException();
 
@@ -185,6 +192,7 @@ namespace skyscraper5.Skyscraper.IO
 
         public static double ReadDouble(this Stream stream)
         {
+			byte[] buffer = new byte[8];
 			if (stream.Read(buffer, 0, 8) != 8)
 		        throw new EndOfStreamException();
 	        if (!BitConverter.IsLittleEndian)

skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs

@@ -6,7 +6,7 @@ using System.Threading.Tasks;
 
 namespace skyscraper8.Skyscraper.Math
 {
-	internal class EntropyCalculatorStream : Stream
+	public class EntropyCalculatorStream : Stream
 	{
 		public override void Flush()
 		{

skyscraper8/Skyscraper/Net/IpTrafficInfo.cs

@@ -33,8 +33,8 @@ namespace skyscraper5.Skyscraper.Net
         public IPAddress Target { get; set; }
 
         public byte Protocol { get; set; }
-		public string SourceName { get; internal set; }
+		public string SourceName { get; set; }
-		public string TargetName { get; internal set; }
+		public string TargetName { get; set; }
 
 		public bool Equals(IpTrafficInfo other)
         {
@@ -104,6 +104,5 @@ namespace skyscraper5.Skyscraper.Net
             sb.Append(")");
             return sb.ToString();
         }
-
 	}
 }

skyscraper8/Skyscraper/Scraper/Dns/IDnsDataSource.cs

@@ -10,6 +10,11 @@ namespace skyscraper5.src.Skyscraper.Scraper.Dns
 	public interface IDnsDataSource
 	{
 		long DnsCountA();
+		/// <summary>
+		/// "SELECT domain_name FROM dns_ips ips LEFT JOIN dns_records records on ips.ip_id = records.ip_id LEFT JOIN dns_domains domains on records.domain_id = domains.domain_id WHERE ips.ip_address = @ia AND (records.record_type = 1 OR records.record_type = 28)"
+		/// </summary>
+		/// <param name="source"></param>
+		/// <returns></returns>
 		string DnsIpToName(IPAddress source);
 		void RememberDnsRecord(DnsRecord record);
 		bool TestForIp(IPAddress iP);

skyscraper8/Skyscraper/Scraper/ISkyscraperUiJunction.cs

@@ -5,6 +5,7 @@ using System.Text;
 using System.Threading.Tasks;
 using skyscraper5.Docsis;
 using skyscraper5.Docsis.MacManagement;
+using skyscraper5.Dvb.DataBroadcasting.IntModel;
 using skyscraper5.Dvb.DataBroadcasting.SkyscraperVfs;
 using skyscraper5.Dvb.Descriptors;
 using skyscraper5.Dvb.Psi.Model;
@@ -17,8 +18,10 @@ using skyscraper5.Skyscraper.IO.CrazycatStreamReader;
 using skyscraper5.Skyscraper.Net;
 using skyscraper5.src.Skyscraper.FrequencyListGenerator;
 using skyscraper5.Teletext.Wss;
+using skyscraper8.GSE;
 using skyscraper8.Skyscraper.Drawing;
 using skyscraper8.Skyscraper.FrequencyListGenerator;
+using skyscraper8.Skyscraper.Scraper;
 
 namespace skyscraper5.Skyscraper.Scraper
 {
@@ -28,7 +31,7 @@ namespace skyscraper5.Skyscraper.Scraper
         void NotifySdtService(SdtService sdtService);
         void NotifyPatProgram(int pmtPid, ushort programId);
         void NotifyPmtProgram(ProgramMapping result, int pmtPid);
-        void NotifyMpeTraffic(IpTrafficInfo iti, int ipv4PacketLength);
+        void NotifyMpeTraffic(IpTrafficInfo iti, byte[] ipv4PacketLength);
         void NotifyAit(AitApplication aitApplication);
         void DsmCcModuleAdd(int elementaryPid, ushort moduleInfoModuleId, byte moduleInfoModuleVersion);
         void DsmCcModuleProgress(int elementaryPid, ushort moduleInfoModuleId, byte moduleInfoModuleVersion, double moduleInfoDownloadProgress);
@@ -42,12 +45,8 @@ namespace skyscraper5.Skyscraper.Scraper
         void NotifyCat(CaDescriptor caDescriptor);
         void NotifyScte35(ushort programNumber, SpliceInsert spliceInsert);
         void NotifyScte35(ushort programNumber, TimeSignal spliceInsert);
-        void SetMemorySaverMode(bool saveMemory);
-        void NotifyDocsisCarrier(DocsisEnvironment docsisEnvironment);
         void NotifyDocsisFrequency(uint? frequency, bool isUpstream, object mmm);
-        void SetGseMode();
         void ShowFramegrab(int currentNetworkId, int transportStreamId, ushort mappingProgramNumber, int mappingStreamElementaryPid, byte[] imageData);
-        void NotifyBlockstreamCarrier();
 
         IEnumerable<HumanReadableService> GetServices();
         void OnBlindscanOpenFoundFrquenciesWindow(List<BlindscanSearchResult> foundFrequencies, STD_TYPE tunerMetadataType);
@@ -187,6 +186,18 @@ namespace skyscraper5.Skyscraper.Scraper
         /// <returns>Returns true if the "Do not auto-zap" checkbox is checked in the FoundFrequenciesWIndow</returns>
         bool MayAutoZap();
 		void NotifyNit(NitNetwork nitNetwork);
+		void EnableUiFeature(SkyscraperUiFeature bbframeAnalysis);
+		void OnIpMacNotification(int sourcePid, Platform platform, Target target, Operational operational);
+		void OnSsuNotification(Dvb.SystemSoftwareUpdate.Model.UpdateNotificationGroup common, Dvb.SystemSoftwareUpdate.Model.UpdateNotificationTarget target, ushort programNumber);
+
+
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="bbHeader"></param>
+        /// <param name="payload">The payload begins at byte 11</param>
+		void OnBbframe(BBHeader bbHeader, byte[] payload);
+		void OnDetectionOfInnerTs(SkyscraperContext child, object identifier);
 
 		TaskQueue Tasks { get; set; }
     }

skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs

@@ -247,6 +247,11 @@ namespace skyscraper5.Skyscraper.Scraper
         }
 
         public void IngestFromStream(Stream stream)
+        {
+            IngestFromStream(stream, true);
+        }
+
+        public void IngestFromStream(Stream stream, bool closeWhenDone = true)
         {
             byte[] buffer = new byte[188];
             while (true)
@@ -267,7 +272,10 @@ namespace skyscraper5.Skyscraper.Scraper
                 }
             }
 
+            if (closeWhenDone)
+            {
                 stream.Close();
+            }
             runningEvents = null;
         }
 
@@ -305,7 +313,7 @@ namespace skyscraper5.Skyscraper.Scraper
                     if (!DvbContext.IsPidProcessorPresent(0x010e))
                     {
                         DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(CreateGsContext()));
-                        UiJunction?.SetGseMode();
+                        UiJunction?.EnableUiFeature(SkyscraperUiFeature.BbframeAnalysis);
                         LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected.");
                         SpecialTsType = 3;
                     }
@@ -327,6 +335,7 @@ namespace skyscraper5.Skyscraper.Scraper
 	        child.IpOutput = this;
 	        child.TsOutput = this;
 	        child.Rcs2Output = this;
+            child.UiJunction = this.UiJunction;
 
             //for futureproofing
             PropertyInfo[] properties = typeof(GsContextDto).GetProperties();
@@ -383,7 +392,7 @@ namespace skyscraper5.Skyscraper.Scraper
                             if (!DvbContext.IsPidProcessorPresent(0x010e))
                             {
                                 DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(CreateGsContext()));
-                                UiJunction?.SetGseMode();
+                                UiJunction?.EnableUiFeature(SkyscraperUiFeature.BbframeAnalysis);
                                 LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected.");
                                 SpecialTsType = 3;
                                 return;
@@ -399,7 +408,7 @@ namespace skyscraper5.Skyscraper.Scraper
                 {
                     docsisPacketProcessor = new DocsisPacketProcessor(this);
                     DvbContext.RegisterPacketProcessor(0x1ffe, docsisPacketProcessor);
-                    UiJunction?.NotifyDocsisCarrier(docsisPacketProcessor.DocsisEnvironment);
+                    UiJunction?.EnableUiFeature(SkyscraperUiFeature.DocsisAnalysis);
                     LogEvent(SkyscraperContextEvent.SpecialTsMode, "DOCSIS Carrier TS detected.");
 					SpecialTsType = 4;
 					return;
@@ -411,7 +420,7 @@ namespace skyscraper5.Skyscraper.Scraper
                 {
                     MultiprotocolEncapsulationDecoder blockstreamDecoder = new MultiprotocolEncapsulationDecoder(this);
                     DvbContext.RegisterPacketProcessor(0x0020, new PsiDecoder(0x0020,blockstreamDecoder));
-                    UiJunction?.NotifyBlockstreamCarrier();
+                    UiJunction?.EnableUiFeature(SkyscraperUiFeature.BlockstreamAnalysis);
                     LogEvent(SkyscraperContextEvent.SpecialTsMode, "Blockstream Carrier TS detected.");
 					SpecialTsType = 5;
 					return;
@@ -1436,6 +1445,8 @@ namespace skyscraper5.Skyscraper.Scraper
 
         public void UpdateNotification(UpdateNotificationGroup common, UpdateNotificationTarget target, ushort ProgramNumber)
         {
+            UiJunction?.OnSsuNotification(common, target, ProgramNumber);
+
             /*int hashCode = target.GetHashCode();
             if (!ScraperStorage.TestForUpdateNotification(hashCode, common))
             {
@@ -1567,9 +1578,7 @@ namespace skyscraper5.Skyscraper.Scraper
             if (isObjectCarousel)
                 throw new NotImplementedException();
 
-            UiJunction?.SetMemorySaverMode(true);
             ObjectStorage.DataCarouselModuleArrival(CurrentNetworkId.Value, CurrentTransportStreamId.Value, elementaryPid, moduleModuleId, moduleModuleVersion, result);
-            UiJunction?.SetMemorySaverMode(false);
 
             LogEvent(SkyscraperContextEvent.ModuleDownloadComplete, String.Format("Module {0}, Version {1}", moduleModuleId, moduleModuleVersion));
 
@@ -1717,6 +1726,7 @@ namespace skyscraper5.Skyscraper.Scraper
         private IpTrafficHandler ipTrafficHandler;
         public void OnIpDatagram(int pid, byte[] payload)
         {
+            UiJunction?.EnableUiFeature(SkyscraperUiFeature.IpTrafficAnalysis);
 	        if (ipTrafficHandler == null)
 	        {
 		        StorageConnectionManager storageConnectionManager = StorageConnectionManager.GetInstance();
@@ -1832,7 +1842,7 @@ namespace skyscraper5.Skyscraper.Scraper
             {
                 LogEvent(SkyscraperContextEvent.LearnDns, String.Format("{0} = {1}", DnsCache.LastLearned.Value, DnsCache.LastLearned.Key));
             }
-			UiJunction?.NotifyMpeTraffic(iti, ipv4Packet.Length);
+			UiJunction?.NotifyMpeTraffic(iti, ipv4Packet);
 
 			if (trafficInfos.Add(iti))
             {
@@ -1993,6 +2003,7 @@ namespace skyscraper5.Skyscraper.Scraper
 
 		        child.InitalizeFilterChain(packetFilters.ToArray());
 		        subSkyscrapers.Add(identifier, child);
+                UiJunction?.OnDetectionOfInnerTs(child, identifier);
 	        }
 
 	        SkyscraperContext context = subSkyscrapers[identifier];
@@ -2185,6 +2196,8 @@ namespace skyscraper5.Skyscraper.Scraper
 
         public void OnIpMacNotification(int sourcePid, Dvb.DataBroadcasting.IntModel.Platform platform, Target target, Operational operational)
         {
+            UiJunction?.OnIpMacNotification(sourcePid, platform, target, operational);
+
             IEnumerable<IpMacNotification> ipMacNotifications = IpMacNotification.FlatMap(platform, target, operational);
             foreach (IpMacNotification notification in ipMacNotifications)
             {

skyscraper8/Skyscraper/Scraper/SkyscraperUiFeature.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.Skyscraper.Scraper
+{
+	public enum SkyscraperUiFeature
+	{
+		BbframeAnalysis,
+		DocsisAnalysis,
+		BlockstreamAnalysis,
+		IpTrafficAnalysis
+	}
+}