diff --git a/skyscraper8/GS/BBframeDeencapsulator3.cs b/skyscraper8/GS/BBframeDeencapsulator3.cs index a827251..079efdb 100644 --- a/skyscraper8/GS/BBframeDeencapsulator3.cs +++ b/skyscraper8/GS/BBframeDeencapsulator3.cs @@ -32,6 +32,7 @@ class BbframeDeencapsulator3 : IBbframeDeencapsulator if (!bbHeader.Valid) return; + if (mis == null) mis = new IMisHandler[256]; if (mis[bbHeader.Matype2] == null) @@ -39,8 +40,9 @@ class BbframeDeencapsulator3 : IBbframeDeencapsulator mis[bbHeader.Matype2] = new MisHandlerProxy(context.MisClone(bbHeader.Matype2)); } - mis[bbHeader.Matype2].PushFrame(bbHeader, new ReadOnlySpan(bbframe, 11, bbframe.Length - 11)); - } + mis[bbHeader.Matype2].PushFrame(bbHeader, new ReadOnlySpan(bbframe, 11, bbframe.Length - 11)); + context.UiJunction.OnBbframe(bbHeader, bbframe); + } private IMisHandler[] mis; } \ No newline at end of file diff --git a/skyscraper8/GS/GsContextDto.cs b/skyscraper8/GS/GsContextDto.cs index feb6d84..d543a60 100644 --- a/skyscraper8/GS/GsContextDto.cs +++ b/skyscraper8/GS/GsContextDto.cs @@ -5,6 +5,7 @@ using System.Reflection; using System.Text; using System.Threading.Tasks; using skyscraper5.Dvb.DataBroadcasting; +using skyscraper5.Skyscraper.Scraper; using skyscraper5.src.InteractionChannel; using skyscraper8.Skyscraper.Scraper; @@ -39,6 +40,7 @@ namespace skyscraper8.GS public IMultiprotocolEncapsulationEventHandler IpOutput { get; set; } public InteractionChannelHandler Rcs2Output { get; set; } + public ISkyscraperUiJunction UiJunction { get; internal set; } private Dictionary trafficTypes; public void InformTrafficType(ushort trafficType) diff --git a/skyscraper8/GS/MisHandlerProxy.cs b/skyscraper8/GS/MisHandlerProxy.cs index b8a3f0f..fc2d4b5 100644 --- a/skyscraper8/GS/MisHandlerProxy.cs +++ b/skyscraper8/GS/MisHandlerProxy.cs @@ -1,166 +1,166 @@ -using log4net; -using skyscraper5.Mpeg2; -using skyscraper8.GS; -using skyscraper8.GS.GSE_BFBS; -using skyscraper8.GS.GSE_RollingSyncByte; -using skyscraper8.GS.SiminnRadiomidun; -using skyscraper8.GSE.GSE_HEM; -using skyscraper8.GSE.GSE; - -namespace skyscraper8.GSE; - -class MisHandlerProxy : IMisHandler -{ - public GsContextDto Context { get; set; } - private readonly ILog logger; - private HashSet> _postedStreamTypes; - - public MisHandlerProxy(GsContextMisDto context) - { - this.logger = LogManager.GetLogger(String.Format("{0} MIS {1}",System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name,context.Mis)); - this.Context = context; - } - - private int state; - private Queue> waitingPackets; - private DateTime lastWaitingTimePrinted; - private IMisHandler misHandler; - private bool unknownStateError; - - - public void PushFrame(BBHeader bbHeader, ReadOnlySpan readOnlySpan) - { - if (readOnlySpan.Length == 0) - { - return; - } - - switch (state) - { - case 0: //No packets collected yet. - waitingPackets = new Queue>(); - logger.InfoFormat("Attempting to detect the stream type of MIS {0}...",bbHeader.Matype2); - waitingPackets.Enqueue(new Tuple(bbHeader, readOnlySpan.ToArray())); - lastWaitingTimePrinted = DateTime.Now; - state = 1; - break; - case 1: - DateTime currentTime = DateTime.Now; - if (currentTime - lastWaitingTimePrinted > TimeSpan.FromSeconds(1)) - { - logger.InfoFormat("Waiting for {0} more packets on MIS {1}...",100 - waitingPackets.Count,bbHeader.Matype2); - lastWaitingTimePrinted = currentTime; - } - waitingPackets.Enqueue(new Tuple(bbHeader, readOnlySpan.ToArray())); - if (waitingPackets.Count == 100) - { - state = 2; - } - break; - case 2: - RunDetection(); - DrainQueue(); - misHandler.PushFrame(bbHeader, readOnlySpan); - state = 3; - break; - case 3: +using log4net; +using skyscraper5.Mpeg2; +using skyscraper8.GS; +using skyscraper8.GS.GSE_BFBS; +using skyscraper8.GS.GSE_RollingSyncByte; +using skyscraper8.GS.SiminnRadiomidun; +using skyscraper8.GSE.GSE_HEM; +using skyscraper8.GSE.GSE; + +namespace skyscraper8.GSE; + +class MisHandlerProxy : IMisHandler +{ + public GsContextDto Context { get; set; } + private readonly ILog logger; + private HashSet> _postedStreamTypes; + + public MisHandlerProxy(GsContextMisDto context) + { + this.logger = LogManager.GetLogger(String.Format("{0} MIS {1}",System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name,context.Mis)); + this.Context = context; + } + + private int state; + private Queue> waitingPackets; + private DateTime lastWaitingTimePrinted; + private IMisHandler misHandler; + private bool unknownStateError; + + + public void PushFrame(BBHeader bbHeader, ReadOnlySpan readOnlySpan) + { + if (readOnlySpan.Length == 0) + { + return; + } + + switch (state) + { + case 0: //No packets collected yet. + waitingPackets = new Queue>(); + logger.InfoFormat("Attempting to detect the stream type of MIS {0}...",bbHeader.Matype2); + waitingPackets.Enqueue(new Tuple(bbHeader, readOnlySpan.ToArray())); + lastWaitingTimePrinted = DateTime.Now; + state = 1; + break; + case 1: + DateTime currentTime = DateTime.Now; + if (currentTime - lastWaitingTimePrinted > TimeSpan.FromSeconds(1)) + { + logger.InfoFormat("Waiting for {0} more packets on MIS {1}...",100 - waitingPackets.Count,bbHeader.Matype2); + lastWaitingTimePrinted = currentTime; + } + waitingPackets.Enqueue(new Tuple(bbHeader, readOnlySpan.ToArray())); + if (waitingPackets.Count == 100) + { + state = 2; + } + break; + case 2: + RunDetection(); + DrainQueue(); + misHandler.PushFrame(bbHeader, readOnlySpan); + state = 3; + break; + case 3: try { misHandler.PushFrame(bbHeader, readOnlySpan); - } - catch (Exception e) + } + catch (Exception e) { logger.ErrorFormat("An error occurred while handling a GS packet. This means that the stream type autodetection guessed wrong."); OnUnknownStreamFormat(); - } - break; - default: - if (!unknownStateError) - { - logger.FatalFormat("The Finite-state machine of the MIS Handler reached an impossible state. This is a bug in skyscraper8. It would be great if you could share a sample of this stream to reproduce this, so I can fix this."); - unknownStateError = true; - } - break; - } - } - - private enum DetectedGsType - { - ContinuousGSE, - GSE_HEM, - SimminRadiomidun, - BfbsGSE, - GseWithRollingSyncByte - } - - private void SucessfulDetection(DetectedGsType detectedGsType) - { - logger.InfoFormat("MIS {0} looks like {1}, will try to treat it as such.",Context.GetMisId(),detectedGsType); - switch (detectedGsType) - { - case DetectedGsType.ContinuousGSE: misHandler = new GseReader(Context); break; - case DetectedGsType.GSE_HEM: misHandler = new GseHemReader(Context); break; - case DetectedGsType.SimminRadiomidun: misHandler = new SiminnRadiomidunReader(Context); break; - case DetectedGsType.BfbsGSE: misHandler = new BfbsGseReader(Context); break; - case DetectedGsType.GseWithRollingSyncByte: misHandler = new GseWithRollingSyncByteReader(Context); break; - default: - logger.FatalFormat("However, {0} is not implemented yet, sorry...", detectedGsType); - misHandler = new NullMisHandler(); - break; - } - } - - private bool RunDetection() - { - //bbHeader.TsGs == 1 && bbHeader.SyncByte == 0 - int continuousGseScore = waitingPackets.Count(x => x.Item1.TsGs == 1 && x.Item1.SyncByte == 0); - if (continuousGseScore == 100) - { - SucessfulDetection(DetectedGsType.ContinuousGSE); - return true; - } - - //bbHeader.TsGs == 2 && bbHeader.SyncByte == 0 - int gseHemScore = waitingPackets.Count(x => x.Item1.TsGs == 2 && x.Item1.SyncByte == 0); - if (gseHemScore == 100) - { - SucessfulDetection(DetectedGsType.GSE_HEM); - return true; - } - - //bbHeader.TsGs == 0 && bbHeader.SyncByte == 71 && bbHeader.UserPacketLength == 188 - int simminRadiomidunScore = waitingPackets.Count(x => x.Item1.TsGs == 0 && x.Item1.SyncByte == 71 && x.Item1.UserPacketLength == 188); - if (simminRadiomidunScore == 100) - { - SucessfulDetection(DetectedGsType.SimminRadiomidun); - return true; - } - - //bbHeader.TsGs == 1 && bbHeader.SyncByte == 1 && bbHeader.UserPacketLength == 0 - int bfbsGseScore = waitingPackets.Count(x => x.Item1.TsGs == 1 && x.Item1.SyncByte == 1 && x.Item1.UserPacketLength == 0); - if (bfbsGseScore == 100) - { - SucessfulDetection(DetectedGsType.BfbsGSE); - return true; - } - - bool allPacketsHaveValidCrc32 = AnalyzeCrc32Values(); - - if (!allPacketsHaveValidCrc32) - { - if (StatisticalAnalysis_Ses12_11476v()) - { - SucessfulDetection(DetectedGsType.GseWithRollingSyncByte); - return true; - } + } + break; + default: + if (!unknownStateError) + { + logger.FatalFormat("The Finite-state machine of the MIS Handler reached an impossible state. This is a bug in skyscraper8. It would be great if you could share a sample of this stream to reproduce this, so I can fix this."); + unknownStateError = true; + } + break; + } + } + + private enum DetectedGsType + { + ContinuousGSE, + GSE_HEM, + SimminRadiomidun, + BfbsGSE, + GseWithRollingSyncByte + } + + private void SucessfulDetection(DetectedGsType detectedGsType) + { + logger.InfoFormat("MIS {0} looks like {1}, will try to treat it as such.",Context.GetMisId(),detectedGsType); + switch (detectedGsType) + { + case DetectedGsType.ContinuousGSE: misHandler = new GseReader(Context); break; + case DetectedGsType.GSE_HEM: misHandler = new GseHemReader(Context); break; + case DetectedGsType.SimminRadiomidun: misHandler = new SiminnRadiomidunReader(Context); break; + case DetectedGsType.BfbsGSE: misHandler = new BfbsGseReader(Context); break; + case DetectedGsType.GseWithRollingSyncByte: misHandler = new GseWithRollingSyncByteReader(Context); break; + default: + logger.FatalFormat("However, {0} is not implemented yet, sorry...", detectedGsType); + misHandler = new NullMisHandler(); + break; + } + } + + private bool RunDetection() + { + //bbHeader.TsGs == 1 && bbHeader.SyncByte == 0 + int continuousGseScore = waitingPackets.Count(x => x.Item1.TsGs == 1 && x.Item1.SyncByte == 0); + if (continuousGseScore == 100) + { + SucessfulDetection(DetectedGsType.ContinuousGSE); + return true; + } + + //bbHeader.TsGs == 2 && bbHeader.SyncByte == 0 + int gseHemScore = waitingPackets.Count(x => x.Item1.TsGs == 2 && x.Item1.SyncByte == 0); + if (gseHemScore == 100) + { + SucessfulDetection(DetectedGsType.GSE_HEM); + return true; + } + + //bbHeader.TsGs == 0 && bbHeader.SyncByte == 71 && bbHeader.UserPacketLength == 188 + int simminRadiomidunScore = waitingPackets.Count(x => x.Item1.TsGs == 0 && x.Item1.SyncByte == 71 && x.Item1.UserPacketLength == 188); + if (simminRadiomidunScore == 100) + { + SucessfulDetection(DetectedGsType.SimminRadiomidun); + return true; + } + + //bbHeader.TsGs == 1 && bbHeader.SyncByte == 1 && bbHeader.UserPacketLength == 0 + int bfbsGseScore = waitingPackets.Count(x => x.Item1.TsGs == 1 && x.Item1.SyncByte == 1 && x.Item1.UserPacketLength == 0); + if (bfbsGseScore == 100) + { + SucessfulDetection(DetectedGsType.BfbsGSE); + return true; + } + + bool allPacketsHaveValidCrc32 = AnalyzeCrc32Values(); + + if (!allPacketsHaveValidCrc32) + { + if (StatisticalAnalysis_Ses12_11476v()) + { + SucessfulDetection(DetectedGsType.GseWithRollingSyncByte); + return true; + } } OnUnknownStreamFormat(); - return false; - } - - private void DrainQueue() - { + return false; + } + + private void DrainQueue() + { try { while (waitingPackets.Count > 0) @@ -169,46 +169,46 @@ class MisHandlerProxy : IMisHandler misHandler.PushFrame(dequeue.Item1, dequeue.Item2); } waitingPackets = null; - } - catch (Exception ex) + } + catch (Exception ex) { logger.ErrorFormat("While trying to handle the buffered GS packets, an error occurred. This means that the stream type autodetection guessed wrong."); OnUnknownStreamFormat(); DrainQueue(); - } - } - + } + } + private void OnUnknownStreamFormat() { logger.WarnFormat("MIS {0} has an unknown stream type. The IP packet detection algorithm from older skyscraper8 versions will be used. For further analysis, please consider submitting a sample of this stream.", Context.GetMisId()); misHandler = new IPv4PacketBruteforce(Context); - } - - private bool AnalyzeCrc32Values() - { - foreach (Tuple waitingPacket in waitingPackets) - { - bool validateCrc = DvbCrc32.ValidateCrc(waitingPacket.Item2); - if (!validateCrc) - return false; - } - - return true; - } - - private bool StatisticalAnalysis_Ses12_11476v() - { - byte[] syncBytes = new byte[256]; - foreach (Tuple dequeue in waitingPackets) - { - syncBytes[dequeue.Item1.SyncByte]++; - if (syncBytes[dequeue.Item1.SyncByte] > 3) - return false; - - if (dequeue.Item1.DataFieldLength != dequeue.Item2.Length) - return false; - } - - return true; - } + } + + private bool AnalyzeCrc32Values() + { + foreach (Tuple waitingPacket in waitingPackets) + { + bool validateCrc = DvbCrc32.ValidateCrc(waitingPacket.Item2); + if (!validateCrc) + return false; + } + + return true; + } + + private bool StatisticalAnalysis_Ses12_11476v() + { + byte[] syncBytes = new byte[256]; + foreach (Tuple dequeue in waitingPackets) + { + syncBytes[dequeue.Item1.SyncByte]++; + if (syncBytes[dequeue.Item1.SyncByte] > 3) + return false; + + if (dequeue.Item1.DataFieldLength != dequeue.Item2.Length) + return false; + } + + return true; + } } \ No newline at end of file diff --git a/skyscraper8/Mpeg2/Psi/Model/ProgramMappingStream.cs b/skyscraper8/Mpeg2/Psi/Model/ProgramMappingStream.cs index dc431c5..fcac3c8 100644 --- a/skyscraper8/Mpeg2/Psi/Model/ProgramMappingStream.cs +++ b/skyscraper8/Mpeg2/Psi/Model/ProgramMappingStream.cs @@ -208,18 +208,22 @@ namespace skyscraper5.Mpeg2.Psi.Model public byte[] AudioSpecifConfig { get; internal set; } public byte[] AudioProfileLevelIndication { get; internal set; } public uint? MetadataApplicationFormatIdentifier { get; internal set; } - public ushort MetadataApplicationFormat { get; internal set; } - public byte MetadataFormat { get; internal set; } + public ushort? MetadataApplicationFormat { get; internal set; } + public byte? MetadataFormat { get; internal set; } public uint? MetadataFormatIdentifier { get; internal set; } - public byte MetadataServiceId { get; internal set; } - public int DecoderConfigFlag { get; internal set; } + public byte? MetadataServiceId { get; internal set; } + public int? DecoderConfigFlag { get; internal set; } public byte[] ServiceIdentificationRecord { get; internal set; } public byte[] DecoderConfig { get; internal set; } public byte[] DecoderConfigIdentificationRecord { get; internal set; } - public ushort DecoderConfigMetadataServiceId { get; internal set; } + public ushort? DecoderConfigMetadataServiceId { get; internal set; } public byte[] Reserved { get; internal set; } public byte[] PrivateData { get; internal set; } + //Voile tools + public bool Ac3DescriptorPresent { get; internal set; } + public int NumDescriptors { get; internal set; } + public List UnknownUserDefines; public ProgramMappingStream(PmtStreamType streamType, int elementaryPid) { diff --git a/skyscraper8/Mpeg2/Psi/PmtParser.cs b/skyscraper8/Mpeg2/Psi/PmtParser.cs index 3011a74..1fa80cc 100644 --- a/skyscraper8/Mpeg2/Psi/PmtParser.cs +++ b/skyscraper8/Mpeg2/Psi/PmtParser.cs @@ -65,7 +65,11 @@ namespace skyscraper5.Mpeg2.Psi ProgramMapping result = new ProgramMapping(programNumber, pcrPid); if (programInfoLength > 0) { - byte[] descriptorBuffer = buffer.ReadBytes(programInfoLength); + byte[] descriptorBuffer = new byte[programInfoLength]; + int descriptorBufferFilling = buffer.Read(descriptorBuffer, 0, programInfoLength); + if (descriptorBufferFilling != programInfoLength) + return; + ParseOuterDescriptors(result, descriptorBuffer); } @@ -97,6 +101,7 @@ namespace skyscraper5.Mpeg2.Psi IEnumerable descriptors = TsDescriptorUnpacker.GetInstance().UnpackDescriptors(inputBuffer, "PMT"); foreach (TsDescriptor dvbDescriptor in descriptors) { + output.NumDescriptors++; switch (dvbDescriptor.GetType().Name) { case nameof(StreamIdentifierDescriptor): @@ -140,6 +145,7 @@ namespace skyscraper5.Mpeg2.Psi output.BSID = ac3Descriptor.BSID; output.ComponentType = ac3Descriptor.ComponentType; output.MainId = ac3Descriptor.MainId; + output.Ac3DescriptorPresent = true; break; case nameof(VideoStreamDescriptor): VideoStreamDescriptor videoStreamDescriptor = (VideoStreamDescriptor)dvbDescriptor; diff --git a/skyscraper8/Skyscraper/IO/StreamExtensions.cs b/skyscraper8/Skyscraper/IO/StreamExtensions.cs index a7c28ee..d0af726 100644 --- a/skyscraper8/Skyscraper/IO/StreamExtensions.cs +++ b/skyscraper8/Skyscraper/IO/StreamExtensions.cs @@ -11,11 +11,10 @@ namespace skyscraper5.Skyscraper.IO { public static class StreamExtensions { - private static byte[] buffer = new byte[8]; - [DebuggerStepThrough] public static ushort ReadUInt16BE(this Stream stream) { + byte[] buffer = new byte[2]; if (stream.Read(buffer, 0, 2) != 2) throw new EndOfStreamException(); if (BitConverter.IsLittleEndian) @@ -25,6 +24,7 @@ namespace skyscraper5.Skyscraper.IO public static short ReadInt16LE(this Stream stream) { + byte[] buffer = new byte[2]; if (stream.Read(buffer, 0, 2) != 2) throw new EndOfStreamException(); if (!BitConverter.IsLittleEndian) @@ -34,7 +34,8 @@ namespace skyscraper5.Skyscraper.IO public static ushort ReadUInt16LE(this Stream stream) { - if (stream.Read(buffer, 0, 2) != 2) + byte[] buffer = new byte[2]; + if (stream.Read(buffer, 0, 2) != 2) throw new EndOfStreamException(); if (!BitConverter.IsLittleEndian) Array.Reverse(buffer, 0, 2); @@ -55,6 +56,7 @@ namespace skyscraper5.Skyscraper.IO [DebuggerStepThrough] public static uint ReadUInt32LE(this Stream stream) { + byte[] buffer = new byte[4]; if (stream.Read(buffer, 0, 4) != 4) throw new EndOfStreamException(); if (!BitConverter.IsLittleEndian) @@ -64,7 +66,8 @@ namespace skyscraper5.Skyscraper.IO public static int ReadInt32LE(this Stream stream) { - if (stream.Read(buffer, 0, 4) != 4) + byte[] buffer = new byte[4]; + if (stream.Read(buffer, 0, 4) != 4) throw new EndOfStreamException(); if (!BitConverter.IsLittleEndian) Array.Reverse(buffer, 0, 4); @@ -82,6 +85,7 @@ namespace skyscraper5.Skyscraper.IO public static long ReadInt64LE(this Stream stream) { + byte[] buffer = new byte[8]; if (stream.Read(buffer, 0, 8) != 8) throw new EndOfStreamException(); if (!BitConverter.IsLittleEndian) @@ -91,6 +95,7 @@ namespace skyscraper5.Skyscraper.IO public static ulong ReadUInt64LE(this Stream stream) { + byte[] buffer = new byte[8]; if (stream.Read(buffer, 0, 8) != 8) throw new EndOfStreamException(); if (!BitConverter.IsLittleEndian) @@ -100,7 +105,8 @@ namespace skyscraper5.Skyscraper.IO public static ulong ReadUInt64BE(this Stream stream) { - if (stream.Read(buffer, 0, 8) != 8) + byte[] buffer = new byte[8]; + if (stream.Read(buffer, 0, 8) != 8) throw new EndOfStreamException(); if (BitConverter.IsLittleEndian) Array.Reverse(buffer, 0, 4); @@ -150,7 +156,8 @@ namespace skyscraper5.Skyscraper.IO [DebuggerStepThrough] public static byte ReadUInt8(this Stream stream) { - if (stream.Read(buffer, 0, 1) != 1) + byte[] buffer = new byte[1]; + if (stream.Read(buffer, 0, 1) != 1) throw new EndOfStreamException(); return buffer[0]; @@ -185,7 +192,8 @@ namespace skyscraper5.Skyscraper.IO public static double ReadDouble(this Stream stream) { - if (stream.Read(buffer, 0, 8) != 8) + byte[] buffer = new byte[8]; + if (stream.Read(buffer, 0, 8) != 8) throw new EndOfStreamException(); if (!BitConverter.IsLittleEndian) Array.Reverse(buffer, 0, 8); diff --git a/skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs b/skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs index fc5e2a0..4ccb27d 100644 --- a/skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs +++ b/skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs @@ -6,7 +6,7 @@ using System.Threading.Tasks; namespace skyscraper8.Skyscraper.Math { - internal class EntropyCalculatorStream : Stream + public class EntropyCalculatorStream : Stream { public override void Flush() { diff --git a/skyscraper8/Skyscraper/Net/IpTrafficInfo.cs b/skyscraper8/Skyscraper/Net/IpTrafficInfo.cs index 319846b..31a83c1 100644 --- a/skyscraper8/Skyscraper/Net/IpTrafficInfo.cs +++ b/skyscraper8/Skyscraper/Net/IpTrafficInfo.cs @@ -33,8 +33,8 @@ namespace skyscraper5.Skyscraper.Net public IPAddress Target { get; set; } public byte Protocol { get; set; } - public string SourceName { get; internal set; } - public string TargetName { get; internal set; } + public string SourceName { get; set; } + public string TargetName { get; set; } public bool Equals(IpTrafficInfo other) { @@ -104,6 +104,5 @@ namespace skyscraper5.Skyscraper.Net sb.Append(")"); return sb.ToString(); } - } } diff --git a/skyscraper8/Skyscraper/Scraper/Dns/IDnsDataSource.cs b/skyscraper8/Skyscraper/Scraper/Dns/IDnsDataSource.cs index d26d1d0..436bdfb 100644 --- a/skyscraper8/Skyscraper/Scraper/Dns/IDnsDataSource.cs +++ b/skyscraper8/Skyscraper/Scraper/Dns/IDnsDataSource.cs @@ -10,6 +10,11 @@ namespace skyscraper5.src.Skyscraper.Scraper.Dns public interface IDnsDataSource { long DnsCountA(); + /// + /// "SELECT domain_name FROM dns_ips ips LEFT JOIN dns_records records on ips.ip_id = records.ip_id LEFT JOIN dns_domains domains on records.domain_id = domains.domain_id WHERE ips.ip_address = @ia AND (records.record_type = 1 OR records.record_type = 28)" + /// + /// + /// string DnsIpToName(IPAddress source); void RememberDnsRecord(DnsRecord record); bool TestForIp(IPAddress iP); diff --git a/skyscraper8/Skyscraper/Scraper/ISkyscraperUiJunction.cs b/skyscraper8/Skyscraper/Scraper/ISkyscraperUiJunction.cs index 1d95ad7..7760835 100644 --- a/skyscraper8/Skyscraper/Scraper/ISkyscraperUiJunction.cs +++ b/skyscraper8/Skyscraper/Scraper/ISkyscraperUiJunction.cs @@ -5,6 +5,7 @@ using System.Text; using System.Threading.Tasks; using skyscraper5.Docsis; using skyscraper5.Docsis.MacManagement; +using skyscraper5.Dvb.DataBroadcasting.IntModel; using skyscraper5.Dvb.DataBroadcasting.SkyscraperVfs; using skyscraper5.Dvb.Descriptors; using skyscraper5.Dvb.Psi.Model; @@ -17,8 +18,10 @@ using skyscraper5.Skyscraper.IO.CrazycatStreamReader; using skyscraper5.Skyscraper.Net; using skyscraper5.src.Skyscraper.FrequencyListGenerator; using skyscraper5.Teletext.Wss; +using skyscraper8.GSE; using skyscraper8.Skyscraper.Drawing; using skyscraper8.Skyscraper.FrequencyListGenerator; +using skyscraper8.Skyscraper.Scraper; namespace skyscraper5.Skyscraper.Scraper { @@ -28,7 +31,7 @@ namespace skyscraper5.Skyscraper.Scraper void NotifySdtService(SdtService sdtService); void NotifyPatProgram(int pmtPid, ushort programId); void NotifyPmtProgram(ProgramMapping result, int pmtPid); - void NotifyMpeTraffic(IpTrafficInfo iti, int ipv4PacketLength); + void NotifyMpeTraffic(IpTrafficInfo iti, byte[] ipv4PacketLength); void NotifyAit(AitApplication aitApplication); void DsmCcModuleAdd(int elementaryPid, ushort moduleInfoModuleId, byte moduleInfoModuleVersion); void DsmCcModuleProgress(int elementaryPid, ushort moduleInfoModuleId, byte moduleInfoModuleVersion, double moduleInfoDownloadProgress); @@ -42,12 +45,8 @@ namespace skyscraper5.Skyscraper.Scraper void NotifyCat(CaDescriptor caDescriptor); void NotifyScte35(ushort programNumber, SpliceInsert spliceInsert); void NotifyScte35(ushort programNumber, TimeSignal spliceInsert); - void SetMemorySaverMode(bool saveMemory); - void NotifyDocsisCarrier(DocsisEnvironment docsisEnvironment); void NotifyDocsisFrequency(uint? frequency, bool isUpstream, object mmm); - void SetGseMode(); void ShowFramegrab(int currentNetworkId, int transportStreamId, ushort mappingProgramNumber, int mappingStreamElementaryPid, byte[] imageData); - void NotifyBlockstreamCarrier(); IEnumerable GetServices(); void OnBlindscanOpenFoundFrquenciesWindow(List foundFrequencies, STD_TYPE tunerMetadataType); @@ -187,6 +186,18 @@ namespace skyscraper5.Skyscraper.Scraper /// Returns true if the "Do not auto-zap" checkbox is checked in the FoundFrequenciesWIndow bool MayAutoZap(); void NotifyNit(NitNetwork nitNetwork); + void EnableUiFeature(SkyscraperUiFeature bbframeAnalysis); + void OnIpMacNotification(int sourcePid, Platform platform, Target target, Operational operational); + void OnSsuNotification(Dvb.SystemSoftwareUpdate.Model.UpdateNotificationGroup common, Dvb.SystemSoftwareUpdate.Model.UpdateNotificationTarget target, ushort programNumber); + + + /// + /// + /// + /// + /// The payload begins at byte 11 + void OnBbframe(BBHeader bbHeader, byte[] payload); + void OnDetectionOfInnerTs(SkyscraperContext child, object identifier); TaskQueue Tasks { get; set; } } diff --git a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs index 3552853..872bb70 100644 --- a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs +++ b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs @@ -247,6 +247,11 @@ namespace skyscraper5.Skyscraper.Scraper } public void IngestFromStream(Stream stream) + { + IngestFromStream(stream, true); + } + + public void IngestFromStream(Stream stream, bool closeWhenDone = true) { byte[] buffer = new byte[188]; while (true) @@ -267,7 +272,10 @@ namespace skyscraper5.Skyscraper.Scraper } } - stream.Close(); + if (closeWhenDone) + { + stream.Close(); + } runningEvents = null; } @@ -305,7 +313,7 @@ namespace skyscraper5.Skyscraper.Scraper if (!DvbContext.IsPidProcessorPresent(0x010e)) { DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(CreateGsContext())); - UiJunction?.SetGseMode(); + UiJunction?.EnableUiFeature(SkyscraperUiFeature.BbframeAnalysis); LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected."); SpecialTsType = 3; } @@ -327,6 +335,7 @@ namespace skyscraper5.Skyscraper.Scraper child.IpOutput = this; child.TsOutput = this; child.Rcs2Output = this; + child.UiJunction = this.UiJunction; //for futureproofing PropertyInfo[] properties = typeof(GsContextDto).GetProperties(); @@ -383,7 +392,7 @@ namespace skyscraper5.Skyscraper.Scraper if (!DvbContext.IsPidProcessorPresent(0x010e)) { DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(CreateGsContext())); - UiJunction?.SetGseMode(); + UiJunction?.EnableUiFeature(SkyscraperUiFeature.BbframeAnalysis); LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected."); SpecialTsType = 3; return; @@ -399,7 +408,7 @@ namespace skyscraper5.Skyscraper.Scraper { docsisPacketProcessor = new DocsisPacketProcessor(this); DvbContext.RegisterPacketProcessor(0x1ffe, docsisPacketProcessor); - UiJunction?.NotifyDocsisCarrier(docsisPacketProcessor.DocsisEnvironment); + UiJunction?.EnableUiFeature(SkyscraperUiFeature.DocsisAnalysis); LogEvent(SkyscraperContextEvent.SpecialTsMode, "DOCSIS Carrier TS detected."); SpecialTsType = 4; return; @@ -411,7 +420,7 @@ namespace skyscraper5.Skyscraper.Scraper { MultiprotocolEncapsulationDecoder blockstreamDecoder = new MultiprotocolEncapsulationDecoder(this); DvbContext.RegisterPacketProcessor(0x0020, new PsiDecoder(0x0020,blockstreamDecoder)); - UiJunction?.NotifyBlockstreamCarrier(); + UiJunction?.EnableUiFeature(SkyscraperUiFeature.BlockstreamAnalysis); LogEvent(SkyscraperContextEvent.SpecialTsMode, "Blockstream Carrier TS detected."); SpecialTsType = 5; return; @@ -1436,6 +1445,8 @@ namespace skyscraper5.Skyscraper.Scraper public void UpdateNotification(UpdateNotificationGroup common, UpdateNotificationTarget target, ushort ProgramNumber) { + UiJunction?.OnSsuNotification(common, target, ProgramNumber); + /*int hashCode = target.GetHashCode(); if (!ScraperStorage.TestForUpdateNotification(hashCode, common)) { @@ -1567,9 +1578,7 @@ namespace skyscraper5.Skyscraper.Scraper if (isObjectCarousel) throw new NotImplementedException(); - UiJunction?.SetMemorySaverMode(true); ObjectStorage.DataCarouselModuleArrival(CurrentNetworkId.Value, CurrentTransportStreamId.Value, elementaryPid, moduleModuleId, moduleModuleVersion, result); - UiJunction?.SetMemorySaverMode(false); LogEvent(SkyscraperContextEvent.ModuleDownloadComplete, String.Format("Module {0}, Version {1}", moduleModuleId, moduleModuleVersion)); @@ -1717,6 +1726,7 @@ namespace skyscraper5.Skyscraper.Scraper private IpTrafficHandler ipTrafficHandler; public void OnIpDatagram(int pid, byte[] payload) { + UiJunction?.EnableUiFeature(SkyscraperUiFeature.IpTrafficAnalysis); if (ipTrafficHandler == null) { StorageConnectionManager storageConnectionManager = StorageConnectionManager.GetInstance(); @@ -1832,7 +1842,7 @@ namespace skyscraper5.Skyscraper.Scraper { LogEvent(SkyscraperContextEvent.LearnDns, String.Format("{0} = {1}", DnsCache.LastLearned.Value, DnsCache.LastLearned.Key)); } - UiJunction?.NotifyMpeTraffic(iti, ipv4Packet.Length); + UiJunction?.NotifyMpeTraffic(iti, ipv4Packet); if (trafficInfos.Add(iti)) { @@ -1993,6 +2003,7 @@ namespace skyscraper5.Skyscraper.Scraper child.InitalizeFilterChain(packetFilters.ToArray()); subSkyscrapers.Add(identifier, child); + UiJunction?.OnDetectionOfInnerTs(child, identifier); } SkyscraperContext context = subSkyscrapers[identifier]; @@ -2185,6 +2196,8 @@ namespace skyscraper5.Skyscraper.Scraper public void OnIpMacNotification(int sourcePid, Dvb.DataBroadcasting.IntModel.Platform platform, Target target, Operational operational) { + UiJunction?.OnIpMacNotification(sourcePid, platform, target, operational); + IEnumerable ipMacNotifications = IpMacNotification.FlatMap(platform, target, operational); foreach (IpMacNotification notification in ipMacNotifications) { diff --git a/skyscraper8/Skyscraper/Scraper/SkyscraperUiFeature.cs b/skyscraper8/Skyscraper/Scraper/SkyscraperUiFeature.cs new file mode 100644 index 0000000..a633791 --- /dev/null +++ b/skyscraper8/Skyscraper/Scraper/SkyscraperUiFeature.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.Skyscraper.Scraper +{ + public enum SkyscraperUiFeature + { + BbframeAnalysis, + DocsisAnalysis, + BlockstreamAnalysis, + IpTrafficAnalysis + } +}