ft/skyscraper8
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Patches for Voile
Some checks failed
🚀 Pack skyscraper8 / make-zip (push) Failing after 47s
Details

Browse Source
This commit is contained in:
feyris-tan 2026-02-22 21:22:27 +01:00
parent 26beed9606
commit 651a6068d9
12 changed files with 286 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
skyscraper8/GS/BBframeDeencapsulator3.cs
View File

@ -32,6 +32,7 @@ class BbframeDeencapsulator3 : IBbframeDeencapsulator
if (!bbHeader.Valid) if (!bbHeader.Valid)
return; return;
if (mis == null) if (mis == null)
mis = new IMisHandler[256]; mis = new IMisHandler[256];
if (mis[bbHeader.Matype2] == null) if (mis[bbHeader.Matype2] == null)
@ -39,8 +40,9 @@ class BbframeDeencapsulator3 : IBbframeDeencapsulator
mis[bbHeader.Matype2] = new MisHandlerProxy(context.MisClone(bbHeader.Matype2)); mis[bbHeader.Matype2] = new MisHandlerProxy(context.MisClone(bbHeader.Matype2));
} }
mis[bbHeader.Matype2].PushFrame(bbHeader, new ReadOnlySpan<byte>(bbframe, 11, bbframe.Length - 11)); mis[bbHeader.Matype2].PushFrame(bbHeader, new ReadOnlySpan<byte>(bbframe, 11, bbframe.Length - 11));
} context.UiJunction.OnBbframe(bbHeader, bbframe);
}
private IMisHandler[] mis; private IMisHandler[] mis;
} }

2
skyscraper8/GS/GsContextDto.cs
View File

@ -5,6 +5,7 @@ using System.Reflection;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using skyscraper5.Dvb.DataBroadcasting; using skyscraper5.Dvb.DataBroadcasting;
using skyscraper5.Skyscraper.Scraper;
using skyscraper5.src.InteractionChannel; using skyscraper5.src.InteractionChannel;
using skyscraper8.Skyscraper.Scraper; using skyscraper8.Skyscraper.Scraper;
@ -39,6 +40,7 @@ namespace skyscraper8.GS
public IMultiprotocolEncapsulationEventHandler IpOutput { get; set; } public IMultiprotocolEncapsulationEventHandler IpOutput { get; set; }
public InteractionChannelHandler Rcs2Output { get; set; } public InteractionChannelHandler Rcs2Output { get; set; }
public ISkyscraperUiJunction UiJunction { get; internal set; }
private Dictionary<ushort, ulong> trafficTypes; private Dictionary<ushort, ulong> trafficTypes;
public void InformTrafficType(ushort trafficType) public void InformTrafficType(ushort trafficType)

376
skyscraper8/GS/MisHandlerProxy.cs
View File

@ -1,166 +1,166 @@
using log4net; using log4net;
using skyscraper5.Mpeg2; using skyscraper5.Mpeg2;
using skyscraper8.GS; using skyscraper8.GS;
using skyscraper8.GS.GSE_BFBS; using skyscraper8.GS.GSE_BFBS;
using skyscraper8.GS.GSE_RollingSyncByte; using skyscraper8.GS.GSE_RollingSyncByte;
using skyscraper8.GS.SiminnRadiomidun; using skyscraper8.GS.SiminnRadiomidun;
using skyscraper8.GSE.GSE_HEM; using skyscraper8.GSE.GSE_HEM;
using skyscraper8.GSE.GSE; using skyscraper8.GSE.GSE;
namespace skyscraper8.GSE; namespace skyscraper8.GSE;
class MisHandlerProxy : IMisHandler class MisHandlerProxy : IMisHandler
{ {
public GsContextDto Context { get; set; } public GsContextDto Context { get; set; }
private readonly ILog logger; private readonly ILog logger;
private HashSet<Tuple<int, byte>> _postedStreamTypes; private HashSet<Tuple<int, byte>> _postedStreamTypes;
public MisHandlerProxy(GsContextMisDto context) public MisHandlerProxy(GsContextMisDto context)
{ {
this.logger = LogManager.GetLogger(String.Format("{0} MIS {1}",System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name,context.Mis)); this.logger = LogManager.GetLogger(String.Format("{0} MIS {1}",System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name,context.Mis));
this.Context = context; this.Context = context;
} }
private int state; private int state;
private Queue<Tuple<BBHeader, byte[]>> waitingPackets; private Queue<Tuple<BBHeader, byte[]>> waitingPackets;
private DateTime lastWaitingTimePrinted; private DateTime lastWaitingTimePrinted;
private IMisHandler misHandler; private IMisHandler misHandler;
private bool unknownStateError; private bool unknownStateError;
public void PushFrame(BBHeader bbHeader, ReadOnlySpan<byte> readOnlySpan) public void PushFrame(BBHeader bbHeader, ReadOnlySpan<byte> readOnlySpan)
{ {
if (readOnlySpan.Length == 0) if (readOnlySpan.Length == 0)
{ {
return; return;
} }
switch (state) switch (state)
{ {
case 0: //No packets collected yet. case 0: //No packets collected yet.
waitingPackets = new Queue<Tuple<BBHeader, byte[]>>(); waitingPackets = new Queue<Tuple<BBHeader, byte[]>>();
logger.InfoFormat("Attempting to detect the stream type of MIS {0}...",bbHeader.Matype2); logger.InfoFormat("Attempting to detect the stream type of MIS {0}...",bbHeader.Matype2);
waitingPackets.Enqueue(new Tuple<BBHeader, byte[]>(bbHeader, readOnlySpan.ToArray())); waitingPackets.Enqueue(new Tuple<BBHeader, byte[]>(bbHeader, readOnlySpan.ToArray()));
lastWaitingTimePrinted = DateTime.Now; lastWaitingTimePrinted = DateTime.Now;
state = 1; state = 1;
break; break;
case 1: case 1:
DateTime currentTime = DateTime.Now; DateTime currentTime = DateTime.Now;
if (currentTime - lastWaitingTimePrinted > TimeSpan.FromSeconds(1)) if (currentTime - lastWaitingTimePrinted > TimeSpan.FromSeconds(1))
{ {
logger.InfoFormat("Waiting for {0} more packets on MIS {1}...",100 - waitingPackets.Count,bbHeader.Matype2); logger.InfoFormat("Waiting for {0} more packets on MIS {1}...",100 - waitingPackets.Count,bbHeader.Matype2);
lastWaitingTimePrinted = currentTime; lastWaitingTimePrinted = currentTime;
} }
waitingPackets.Enqueue(new Tuple<BBHeader, byte[]>(bbHeader, readOnlySpan.ToArray())); waitingPackets.Enqueue(new Tuple<BBHeader, byte[]>(bbHeader, readOnlySpan.ToArray()));
if (waitingPackets.Count == 100) if (waitingPackets.Count == 100)
{ {
state = 2; state = 2;
} }
break; break;
case 2: case 2:
RunDetection(); RunDetection();
DrainQueue(); DrainQueue();
misHandler.PushFrame(bbHeader, readOnlySpan); misHandler.PushFrame(bbHeader, readOnlySpan);
state = 3; state = 3;
break; break;
case 3: case 3:
try try
{ {
misHandler.PushFrame(bbHeader, readOnlySpan); misHandler.PushFrame(bbHeader, readOnlySpan);
} }
catch (Exception e) catch (Exception e)
{ {
logger.ErrorFormat("An error occurred while handling a GS packet. This means that the stream type autodetection guessed wrong."); logger.ErrorFormat("An error occurred while handling a GS packet. This means that the stream type autodetection guessed wrong.");
OnUnknownStreamFormat(); OnUnknownStreamFormat();
} }
break; break;
default: default:
if (!unknownStateError) if (!unknownStateError)
{ {
logger.FatalFormat("The Finite-state machine of the MIS Handler reached an impossible state. This is a bug in skyscraper8. It would be great if you could share a sample of this stream to reproduce this, so I can fix this."); logger.FatalFormat("The Finite-state machine of the MIS Handler reached an impossible state. This is a bug in skyscraper8. It would be great if you could share a sample of this stream to reproduce this, so I can fix this.");
unknownStateError = true; unknownStateError = true;
} }
break; break;
} }
} }
private enum DetectedGsType private enum DetectedGsType
{ {
ContinuousGSE, ContinuousGSE,
GSE_HEM, GSE_HEM,
SimminRadiomidun, SimminRadiomidun,
BfbsGSE, BfbsGSE,
GseWithRollingSyncByte GseWithRollingSyncByte
} }
private void SucessfulDetection(DetectedGsType detectedGsType) private void SucessfulDetection(DetectedGsType detectedGsType)
{ {
logger.InfoFormat("MIS {0} looks like {1}, will try to treat it as such.",Context.GetMisId(),detectedGsType); logger.InfoFormat("MIS {0} looks like {1}, will try to treat it as such.",Context.GetMisId(),detectedGsType);
switch (detectedGsType) switch (detectedGsType)
{ {
case DetectedGsType.ContinuousGSE: misHandler = new GseReader(Context); break; case DetectedGsType.ContinuousGSE: misHandler = new GseReader(Context); break;
case DetectedGsType.GSE_HEM: misHandler = new GseHemReader(Context); break; case DetectedGsType.GSE_HEM: misHandler = new GseHemReader(Context); break;
case DetectedGsType.SimminRadiomidun: misHandler = new SiminnRadiomidunReader(Context); break; case DetectedGsType.SimminRadiomidun: misHandler = new SiminnRadiomidunReader(Context); break;
case DetectedGsType.BfbsGSE: misHandler = new BfbsGseReader(Context); break; case DetectedGsType.BfbsGSE: misHandler = new BfbsGseReader(Context); break;
case DetectedGsType.GseWithRollingSyncByte: misHandler = new GseWithRollingSyncByteReader(Context); break; case DetectedGsType.GseWithRollingSyncByte: misHandler = new GseWithRollingSyncByteReader(Context); break;
default: default:
logger.FatalFormat("However, {0} is not implemented yet, sorry...", detectedGsType); logger.FatalFormat("However, {0} is not implemented yet, sorry...", detectedGsType);
misHandler = new NullMisHandler(); misHandler = new NullMisHandler();
break; break;
} }
} }
private bool RunDetection() private bool RunDetection()
{ {
//bbHeader.TsGs == 1 && bbHeader.SyncByte == 0 //bbHeader.TsGs == 1 && bbHeader.SyncByte == 0
int continuousGseScore = waitingPackets.Count(x => x.Item1.TsGs == 1 && x.Item1.SyncByte == 0); int continuousGseScore = waitingPackets.Count(x => x.Item1.TsGs == 1 && x.Item1.SyncByte == 0);
if (continuousGseScore == 100) if (continuousGseScore == 100)
{ {
SucessfulDetection(DetectedGsType.ContinuousGSE); SucessfulDetection(DetectedGsType.ContinuousGSE);
return true; return true;
} }
//bbHeader.TsGs == 2 && bbHeader.SyncByte == 0 //bbHeader.TsGs == 2 && bbHeader.SyncByte == 0
int gseHemScore = waitingPackets.Count(x => x.Item1.TsGs == 2 && x.Item1.SyncByte == 0); int gseHemScore = waitingPackets.Count(x => x.Item1.TsGs == 2 && x.Item1.SyncByte == 0);
if (gseHemScore == 100) if (gseHemScore == 100)
{ {
SucessfulDetection(DetectedGsType.GSE_HEM); SucessfulDetection(DetectedGsType.GSE_HEM);
return true; return true;
} }
//bbHeader.TsGs == 0 && bbHeader.SyncByte == 71 && bbHeader.UserPacketLength == 188 //bbHeader.TsGs == 0 && bbHeader.SyncByte == 71 && bbHeader.UserPacketLength == 188
int simminRadiomidunScore = waitingPackets.Count(x => x.Item1.TsGs == 0 && x.Item1.SyncByte == 71 && x.Item1.UserPacketLength == 188); int simminRadiomidunScore = waitingPackets.Count(x => x.Item1.TsGs == 0 && x.Item1.SyncByte == 71 && x.Item1.UserPacketLength == 188);
if (simminRadiomidunScore == 100) if (simminRadiomidunScore == 100)
{ {
SucessfulDetection(DetectedGsType.SimminRadiomidun); SucessfulDetection(DetectedGsType.SimminRadiomidun);
return true; return true;
} }
//bbHeader.TsGs == 1 && bbHeader.SyncByte == 1 && bbHeader.UserPacketLength == 0 //bbHeader.TsGs == 1 && bbHeader.SyncByte == 1 && bbHeader.UserPacketLength == 0
int bfbsGseScore = waitingPackets.Count(x => x.Item1.TsGs == 1 && x.Item1.SyncByte == 1 && x.Item1.UserPacketLength == 0); int bfbsGseScore = waitingPackets.Count(x => x.Item1.TsGs == 1 && x.Item1.SyncByte == 1 && x.Item1.UserPacketLength == 0);
if (bfbsGseScore == 100) if (bfbsGseScore == 100)
{ {
SucessfulDetection(DetectedGsType.BfbsGSE); SucessfulDetection(DetectedGsType.BfbsGSE);
return true; return true;
} }
bool allPacketsHaveValidCrc32 = AnalyzeCrc32Values(); bool allPacketsHaveValidCrc32 = AnalyzeCrc32Values();
if (!allPacketsHaveValidCrc32) if (!allPacketsHaveValidCrc32)
{ {
if (StatisticalAnalysis_Ses12_11476v()) if (StatisticalAnalysis_Ses12_11476v())
{ {
SucessfulDetection(DetectedGsType.GseWithRollingSyncByte); SucessfulDetection(DetectedGsType.GseWithRollingSyncByte);
return true; return true;
} }
} }
OnUnknownStreamFormat(); OnUnknownStreamFormat();
return false; return false;
} }
private void DrainQueue() private void DrainQueue()
{ {
try try
{ {
while (waitingPackets.Count > 0) while (waitingPackets.Count > 0)
@ -169,46 +169,46 @@ class MisHandlerProxy : IMisHandler
misHandler.PushFrame(dequeue.Item1, dequeue.Item2); misHandler.PushFrame(dequeue.Item1, dequeue.Item2);
} }
waitingPackets = null; waitingPackets = null;
} }
catch (Exception ex) catch (Exception ex)
{ {
logger.ErrorFormat("While trying to handle the buffered GS packets, an error occurred. This means that the stream type autodetection guessed wrong."); logger.ErrorFormat("While trying to handle the buffered GS packets, an error occurred. This means that the stream type autodetection guessed wrong.");
OnUnknownStreamFormat(); OnUnknownStreamFormat();
DrainQueue(); DrainQueue();
} }
} }
private void OnUnknownStreamFormat() private void OnUnknownStreamFormat()
{ {
logger.WarnFormat("MIS {0} has an unknown stream type. The IP packet detection algorithm from older skyscraper8 versions will be used. For further analysis, please consider submitting a sample of this stream.", Context.GetMisId()); logger.WarnFormat("MIS {0} has an unknown stream type. The IP packet detection algorithm from older skyscraper8 versions will be used. For further analysis, please consider submitting a sample of this stream.", Context.GetMisId());
misHandler = new IPv4PacketBruteforce(Context); misHandler = new IPv4PacketBruteforce(Context);
} }
private bool AnalyzeCrc32Values() private bool AnalyzeCrc32Values()
{ {
foreach (Tuple<BBHeader, byte[]> waitingPacket in waitingPackets) foreach (Tuple<BBHeader, byte[]> waitingPacket in waitingPackets)
{ {
bool validateCrc = DvbCrc32.ValidateCrc(waitingPacket.Item2); bool validateCrc = DvbCrc32.ValidateCrc(waitingPacket.Item2);
if (!validateCrc) if (!validateCrc)
return false; return false;
} }
return true; return true;
} }
private bool StatisticalAnalysis_Ses12_11476v() private bool StatisticalAnalysis_Ses12_11476v()
{ {
byte[] syncBytes = new byte[256]; byte[] syncBytes = new byte[256];
foreach (Tuple<BBHeader, byte[]> dequeue in waitingPackets) foreach (Tuple<BBHeader, byte[]> dequeue in waitingPackets)
{ {
syncBytes[dequeue.Item1.SyncByte]++; syncBytes[dequeue.Item1.SyncByte]++;
if (syncBytes[dequeue.Item1.SyncByte] > 3) if (syncBytes[dequeue.Item1.SyncByte] > 3)
return false; return false;
if (dequeue.Item1.DataFieldLength != dequeue.Item2.Length) if (dequeue.Item1.DataFieldLength != dequeue.Item2.Length)
return false; return false;
} }
return true; return true;
} }
} }

14
skyscraper8/Mpeg2/Psi/Model/ProgramMappingStream.cs
View File

@ -208,18 +208,22 @@ namespace skyscraper5.Mpeg2.Psi.Model
public byte[] AudioSpecifConfig { get; internal set; } public byte[] AudioSpecifConfig { get; internal set; }
public byte[] AudioProfileLevelIndication { get; internal set; } public byte[] AudioProfileLevelIndication { get; internal set; }
public uint? MetadataApplicationFormatIdentifier { get; internal set; } public uint? MetadataApplicationFormatIdentifier { get; internal set; }
public ushort MetadataApplicationFormat { get; internal set; } public ushort? MetadataApplicationFormat { get; internal set; }
public byte MetadataFormat { get; internal set; } public byte? MetadataFormat { get; internal set; }
public uint? MetadataFormatIdentifier { get; internal set; } public uint? MetadataFormatIdentifier { get; internal set; }
public byte MetadataServiceId { get; internal set; } public byte? MetadataServiceId { get; internal set; }
public int DecoderConfigFlag { get; internal set; } public int? DecoderConfigFlag { get; internal set; }
public byte[] ServiceIdentificationRecord { get; internal set; } public byte[] ServiceIdentificationRecord { get; internal set; }
public byte[] DecoderConfig { get; internal set; } public byte[] DecoderConfig { get; internal set; }
public byte[] DecoderConfigIdentificationRecord { get; internal set; } public byte[] DecoderConfigIdentificationRecord { get; internal set; }
public ushort DecoderConfigMetadataServiceId { get; internal set; } public ushort? DecoderConfigMetadataServiceId { get; internal set; }
public byte[] Reserved { get; internal set; } public byte[] Reserved { get; internal set; }
public byte[] PrivateData { get; internal set; } public byte[] PrivateData { get; internal set; }
//Voile tools
public bool Ac3DescriptorPresent { get; internal set; }
public int NumDescriptors { get; internal set; }
public List<UserDefinedDescriptor> UnknownUserDefines; public List<UserDefinedDescriptor> UnknownUserDefines;
public ProgramMappingStream(PmtStreamType streamType, int elementaryPid) public ProgramMappingStream(PmtStreamType streamType, int elementaryPid)
{ {

8
skyscraper8/Mpeg2/Psi/PmtParser.cs
View File

@ -65,7 +65,11 @@ namespace skyscraper5.Mpeg2.Psi
ProgramMapping result = new ProgramMapping(programNumber, pcrPid); ProgramMapping result = new ProgramMapping(programNumber, pcrPid);
if (programInfoLength > 0) if (programInfoLength > 0)
{ {
byte[] descriptorBuffer = buffer.ReadBytes(programInfoLength); byte[] descriptorBuffer = new byte[programInfoLength];
int descriptorBufferFilling = buffer.Read(descriptorBuffer, 0, programInfoLength);
if (descriptorBufferFilling != programInfoLength)
return;
ParseOuterDescriptors(result, descriptorBuffer); ParseOuterDescriptors(result, descriptorBuffer);
} }
@ -97,6 +101,7 @@ namespace skyscraper5.Mpeg2.Psi
IEnumerable<TsDescriptor> descriptors = TsDescriptorUnpacker.GetInstance().UnpackDescriptors(inputBuffer, "PMT"); IEnumerable<TsDescriptor> descriptors = TsDescriptorUnpacker.GetInstance().UnpackDescriptors(inputBuffer, "PMT");
foreach (TsDescriptor dvbDescriptor in descriptors) foreach (TsDescriptor dvbDescriptor in descriptors)
{ {
output.NumDescriptors++;
switch (dvbDescriptor.GetType().Name) switch (dvbDescriptor.GetType().Name)
{ {
case nameof(StreamIdentifierDescriptor): case nameof(StreamIdentifierDescriptor):
@ -140,6 +145,7 @@ namespace skyscraper5.Mpeg2.Psi
output.BSID = ac3Descriptor.BSID; output.BSID = ac3Descriptor.BSID;
output.ComponentType = ac3Descriptor.ComponentType; output.ComponentType = ac3Descriptor.ComponentType;
output.MainId = ac3Descriptor.MainId; output.MainId = ac3Descriptor.MainId;
output.Ac3DescriptorPresent = true;
break; break;
case nameof(VideoStreamDescriptor): case nameof(VideoStreamDescriptor):
VideoStreamDescriptor videoStreamDescriptor = (VideoStreamDescriptor)dvbDescriptor; VideoStreamDescriptor videoStreamDescriptor = (VideoStreamDescriptor)dvbDescriptor;

22
skyscraper8/Skyscraper/IO/StreamExtensions.cs
View File

@ -11,11 +11,10 @@ namespace skyscraper5.Skyscraper.IO
{ {
public static class StreamExtensions public static class StreamExtensions
{ {
private static byte[] buffer = new byte[8];
[DebuggerStepThrough] [DebuggerStepThrough]
public static ushort ReadUInt16BE(this Stream stream) public static ushort ReadUInt16BE(this Stream stream)
{ {
byte[] buffer = new byte[2];
if (stream.Read(buffer, 0, 2) != 2) if (stream.Read(buffer, 0, 2) != 2)
throw new EndOfStreamException(); throw new EndOfStreamException();
if (BitConverter.IsLittleEndian) if (BitConverter.IsLittleEndian)
@ -25,6 +24,7 @@ namespace skyscraper5.Skyscraper.IO
public static short ReadInt16LE(this Stream stream) public static short ReadInt16LE(this Stream stream)
{ {
byte[] buffer = new byte[2];
if (stream.Read(buffer, 0, 2) != 2) if (stream.Read(buffer, 0, 2) != 2)
throw new EndOfStreamException(); throw new EndOfStreamException();
if (!BitConverter.IsLittleEndian) if (!BitConverter.IsLittleEndian)
@ -34,7 +34,8 @@ namespace skyscraper5.Skyscraper.IO
public static ushort ReadUInt16LE(this Stream stream) public static ushort ReadUInt16LE(this Stream stream)
{ {
if (stream.Read(buffer, 0, 2) != 2) byte[] buffer = new byte[2];
if (stream.Read(buffer, 0, 2) != 2)
throw new EndOfStreamException(); throw new EndOfStreamException();
if (!BitConverter.IsLittleEndian) if (!BitConverter.IsLittleEndian)
Array.Reverse(buffer, 0, 2); Array.Reverse(buffer, 0, 2);
@ -55,6 +56,7 @@ namespace skyscraper5.Skyscraper.IO
[DebuggerStepThrough] [DebuggerStepThrough]
public static uint ReadUInt32LE(this Stream stream) public static uint ReadUInt32LE(this Stream stream)
{ {
byte[] buffer = new byte[4];
if (stream.Read(buffer, 0, 4) != 4) if (stream.Read(buffer, 0, 4) != 4)
throw new EndOfStreamException(); throw new EndOfStreamException();
if (!BitConverter.IsLittleEndian) if (!BitConverter.IsLittleEndian)
@ -64,7 +66,8 @@ namespace skyscraper5.Skyscraper.IO
public static int ReadInt32LE(this Stream stream) public static int ReadInt32LE(this Stream stream)
{ {
if (stream.Read(buffer, 0, 4) != 4) byte[] buffer = new byte[4];
if (stream.Read(buffer, 0, 4) != 4)
throw new EndOfStreamException(); throw new EndOfStreamException();
if (!BitConverter.IsLittleEndian) if (!BitConverter.IsLittleEndian)
Array.Reverse(buffer, 0, 4); Array.Reverse(buffer, 0, 4);
@ -82,6 +85,7 @@ namespace skyscraper5.Skyscraper.IO
public static long ReadInt64LE(this Stream stream) public static long ReadInt64LE(this Stream stream)
{ {
byte[] buffer = new byte[8];
if (stream.Read(buffer, 0, 8) != 8) if (stream.Read(buffer, 0, 8) != 8)
throw new EndOfStreamException(); throw new EndOfStreamException();
if (!BitConverter.IsLittleEndian) if (!BitConverter.IsLittleEndian)
@ -91,6 +95,7 @@ namespace skyscraper5.Skyscraper.IO
public static ulong ReadUInt64LE(this Stream stream) public static ulong ReadUInt64LE(this Stream stream)
{ {
byte[] buffer = new byte[8];
if (stream.Read(buffer, 0, 8) != 8) if (stream.Read(buffer, 0, 8) != 8)
throw new EndOfStreamException(); throw new EndOfStreamException();
if (!BitConverter.IsLittleEndian) if (!BitConverter.IsLittleEndian)
@ -100,7 +105,8 @@ namespace skyscraper5.Skyscraper.IO
public static ulong ReadUInt64BE(this Stream stream) public static ulong ReadUInt64BE(this Stream stream)
{ {
if (stream.Read(buffer, 0, 8) != 8) byte[] buffer = new byte[8];
if (stream.Read(buffer, 0, 8) != 8)
throw new EndOfStreamException(); throw new EndOfStreamException();
if (BitConverter.IsLittleEndian) if (BitConverter.IsLittleEndian)
Array.Reverse(buffer, 0, 4); Array.Reverse(buffer, 0, 4);
@ -150,7 +156,8 @@ namespace skyscraper5.Skyscraper.IO
[DebuggerStepThrough] [DebuggerStepThrough]
public static byte ReadUInt8(this Stream stream) public static byte ReadUInt8(this Stream stream)
{ {
if (stream.Read(buffer, 0, 1) != 1) byte[] buffer = new byte[1];
if (stream.Read(buffer, 0, 1) != 1)
throw new EndOfStreamException(); throw new EndOfStreamException();
return buffer[0]; return buffer[0];
@ -185,7 +192,8 @@ namespace skyscraper5.Skyscraper.IO
public static double ReadDouble(this Stream stream) public static double ReadDouble(this Stream stream)
{ {
if (stream.Read(buffer, 0, 8) != 8) byte[] buffer = new byte[8];
if (stream.Read(buffer, 0, 8) != 8)
throw new EndOfStreamException(); throw new EndOfStreamException();
if (!BitConverter.IsLittleEndian) if (!BitConverter.IsLittleEndian)
Array.Reverse(buffer, 0, 8); Array.Reverse(buffer, 0, 8);

2
skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs
View File

@ -6,7 +6,7 @@ using System.Threading.Tasks;
namespace skyscraper8.Skyscraper.Math namespace skyscraper8.Skyscraper.Math
{ {
internal class EntropyCalculatorStream : Stream public class EntropyCalculatorStream : Stream
{ {
public override void Flush() public override void Flush()
{ {

5
skyscraper8/Skyscraper/Net/IpTrafficInfo.cs
View File

@ -33,8 +33,8 @@ namespace skyscraper5.Skyscraper.Net
public IPAddress Target { get; set; } public IPAddress Target { get; set; }
public byte Protocol { get; set; } public byte Protocol { get; set; }
public string SourceName { get; internal set; } public string SourceName { get; set; }
public string TargetName { get; internal set; } public string TargetName { get; set; }
public bool Equals(IpTrafficInfo other) public bool Equals(IpTrafficInfo other)
{ {
@ -104,6 +104,5 @@ namespace skyscraper5.Skyscraper.Net
sb.Append(")"); sb.Append(")");
return sb.ToString(); return sb.ToString();
} }
} }
} }

5
skyscraper8/Skyscraper/Scraper/Dns/IDnsDataSource.cs
View File

@ -10,6 +10,11 @@ namespace skyscraper5.src.Skyscraper.Scraper.Dns
public interface IDnsDataSource public interface IDnsDataSource
{ {
long DnsCountA(); long DnsCountA();
/// <summary>
/// "SELECT domain_name FROM dns_ips ips LEFT JOIN dns_records records on ips.ip_id = records.ip_id LEFT JOIN dns_domains domains on records.domain_id = domains.domain_id WHERE ips.ip_address = @ia AND (records.record_type = 1 OR records.record_type = 28)"
/// </summary>
/// <param name="source"></param>
/// <returns></returns>
string DnsIpToName(IPAddress source); string DnsIpToName(IPAddress source);
void RememberDnsRecord(DnsRecord record); void RememberDnsRecord(DnsRecord record);
bool TestForIp(IPAddress iP); bool TestForIp(IPAddress iP);

21
skyscraper8/Skyscraper/Scraper/ISkyscraperUiJunction.cs
View File

@ -5,6 +5,7 @@ using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using skyscraper5.Docsis; using skyscraper5.Docsis;
using skyscraper5.Docsis.MacManagement; using skyscraper5.Docsis.MacManagement;
using skyscraper5.Dvb.DataBroadcasting.IntModel;
using skyscraper5.Dvb.DataBroadcasting.SkyscraperVfs; using skyscraper5.Dvb.DataBroadcasting.SkyscraperVfs;
using skyscraper5.Dvb.Descriptors; using skyscraper5.Dvb.Descriptors;
using skyscraper5.Dvb.Psi.Model; using skyscraper5.Dvb.Psi.Model;
@ -17,8 +18,10 @@ using skyscraper5.Skyscraper.IO.CrazycatStreamReader;
using skyscraper5.Skyscraper.Net; using skyscraper5.Skyscraper.Net;
using skyscraper5.src.Skyscraper.FrequencyListGenerator; using skyscraper5.src.Skyscraper.FrequencyListGenerator;
using skyscraper5.Teletext.Wss; using skyscraper5.Teletext.Wss;
using skyscraper8.GSE;
using skyscraper8.Skyscraper.Drawing; using skyscraper8.Skyscraper.Drawing;
using skyscraper8.Skyscraper.FrequencyListGenerator; using skyscraper8.Skyscraper.FrequencyListGenerator;
using skyscraper8.Skyscraper.Scraper;
namespace skyscraper5.Skyscraper.Scraper namespace skyscraper5.Skyscraper.Scraper
{ {
@ -28,7 +31,7 @@ namespace skyscraper5.Skyscraper.Scraper
void NotifySdtService(SdtService sdtService); void NotifySdtService(SdtService sdtService);
void NotifyPatProgram(int pmtPid, ushort programId); void NotifyPatProgram(int pmtPid, ushort programId);
void NotifyPmtProgram(ProgramMapping result, int pmtPid); void NotifyPmtProgram(ProgramMapping result, int pmtPid);
void NotifyMpeTraffic(IpTrafficInfo iti, int ipv4PacketLength); void NotifyMpeTraffic(IpTrafficInfo iti, byte[] ipv4PacketLength);
void NotifyAit(AitApplication aitApplication); void NotifyAit(AitApplication aitApplication);
void DsmCcModuleAdd(int elementaryPid, ushort moduleInfoModuleId, byte moduleInfoModuleVersion); void DsmCcModuleAdd(int elementaryPid, ushort moduleInfoModuleId, byte moduleInfoModuleVersion);
void DsmCcModuleProgress(int elementaryPid, ushort moduleInfoModuleId, byte moduleInfoModuleVersion, double moduleInfoDownloadProgress); void DsmCcModuleProgress(int elementaryPid, ushort moduleInfoModuleId, byte moduleInfoModuleVersion, double moduleInfoDownloadProgress);
@ -42,12 +45,8 @@ namespace skyscraper5.Skyscraper.Scraper
void NotifyCat(CaDescriptor caDescriptor); void NotifyCat(CaDescriptor caDescriptor);
void NotifyScte35(ushort programNumber, SpliceInsert spliceInsert); void NotifyScte35(ushort programNumber, SpliceInsert spliceInsert);
void NotifyScte35(ushort programNumber, TimeSignal spliceInsert); void NotifyScte35(ushort programNumber, TimeSignal spliceInsert);
void SetMemorySaverMode(bool saveMemory);
void NotifyDocsisCarrier(DocsisEnvironment docsisEnvironment);
void NotifyDocsisFrequency(uint? frequency, bool isUpstream, object mmm); void NotifyDocsisFrequency(uint? frequency, bool isUpstream, object mmm);
void SetGseMode();
void ShowFramegrab(int currentNetworkId, int transportStreamId, ushort mappingProgramNumber, int mappingStreamElementaryPid, byte[] imageData); void ShowFramegrab(int currentNetworkId, int transportStreamId, ushort mappingProgramNumber, int mappingStreamElementaryPid, byte[] imageData);
void NotifyBlockstreamCarrier();
IEnumerable<HumanReadableService> GetServices(); IEnumerable<HumanReadableService> GetServices();
void OnBlindscanOpenFoundFrquenciesWindow(List<BlindscanSearchResult> foundFrequencies, STD_TYPE tunerMetadataType); void OnBlindscanOpenFoundFrquenciesWindow(List<BlindscanSearchResult> foundFrequencies, STD_TYPE tunerMetadataType);
@ -187,6 +186,18 @@ namespace skyscraper5.Skyscraper.Scraper
/// <returns>Returns true if the "Do not auto-zap" checkbox is checked in the FoundFrequenciesWIndow</returns> /// <returns>Returns true if the "Do not auto-zap" checkbox is checked in the FoundFrequenciesWIndow</returns>
bool MayAutoZap(); bool MayAutoZap();
void NotifyNit(NitNetwork nitNetwork); void NotifyNit(NitNetwork nitNetwork);
void EnableUiFeature(SkyscraperUiFeature bbframeAnalysis);
void OnIpMacNotification(int sourcePid, Platform platform, Target target, Operational operational);
void OnSsuNotification(Dvb.SystemSoftwareUpdate.Model.UpdateNotificationGroup common, Dvb.SystemSoftwareUpdate.Model.UpdateNotificationTarget target, ushort programNumber);
/// <summary>
///
/// </summary>
/// <param name="bbHeader"></param>
/// <param name="payload">The payload begins at byte 11</param>
void OnBbframe(BBHeader bbHeader, byte[] payload);
void OnDetectionOfInnerTs(SkyscraperContext child, object identifier);
TaskQueue Tasks { get; set; } TaskQueue Tasks { get; set; }
} }

29
skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs
View File

@ -247,6 +247,11 @@ namespace skyscraper5.Skyscraper.Scraper
} }
public void IngestFromStream(Stream stream) public void IngestFromStream(Stream stream)
{
IngestFromStream(stream, true);
}
public void IngestFromStream(Stream stream, bool closeWhenDone = true)
{ {
byte[] buffer = new byte[188]; byte[] buffer = new byte[188];
while (true) while (true)
@ -267,7 +272,10 @@ namespace skyscraper5.Skyscraper.Scraper
} }
} }
stream.Close(); if (closeWhenDone)
{
stream.Close();
}
runningEvents = null; runningEvents = null;
} }
@ -305,7 +313,7 @@ namespace skyscraper5.Skyscraper.Scraper
if (!DvbContext.IsPidProcessorPresent(0x010e)) if (!DvbContext.IsPidProcessorPresent(0x010e))
{ {
DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(CreateGsContext())); DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(CreateGsContext()));
UiJunction?.SetGseMode(); UiJunction?.EnableUiFeature(SkyscraperUiFeature.BbframeAnalysis);
LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected."); LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected.");
SpecialTsType = 3; SpecialTsType = 3;
} }
@ -327,6 +335,7 @@ namespace skyscraper5.Skyscraper.Scraper
child.IpOutput = this; child.IpOutput = this;
child.TsOutput = this; child.TsOutput = this;
child.Rcs2Output = this; child.Rcs2Output = this;
child.UiJunction = this.UiJunction;
//for futureproofing //for futureproofing
PropertyInfo[] properties = typeof(GsContextDto).GetProperties(); PropertyInfo[] properties = typeof(GsContextDto).GetProperties();
@ -383,7 +392,7 @@ namespace skyscraper5.Skyscraper.Scraper
if (!DvbContext.IsPidProcessorPresent(0x010e)) if (!DvbContext.IsPidProcessorPresent(0x010e))
{ {
DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(CreateGsContext())); DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(CreateGsContext()));
UiJunction?.SetGseMode(); UiJunction?.EnableUiFeature(SkyscraperUiFeature.BbframeAnalysis);
LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected."); LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected.");
SpecialTsType = 3; SpecialTsType = 3;
return; return;
@ -399,7 +408,7 @@ namespace skyscraper5.Skyscraper.Scraper
{ {
docsisPacketProcessor = new DocsisPacketProcessor(this); docsisPacketProcessor = new DocsisPacketProcessor(this);
DvbContext.RegisterPacketProcessor(0x1ffe, docsisPacketProcessor); DvbContext.RegisterPacketProcessor(0x1ffe, docsisPacketProcessor);
UiJunction?.NotifyDocsisCarrier(docsisPacketProcessor.DocsisEnvironment); UiJunction?.EnableUiFeature(SkyscraperUiFeature.DocsisAnalysis);
LogEvent(SkyscraperContextEvent.SpecialTsMode, "DOCSIS Carrier TS detected."); LogEvent(SkyscraperContextEvent.SpecialTsMode, "DOCSIS Carrier TS detected.");
SpecialTsType = 4; SpecialTsType = 4;
return; return;
@ -411,7 +420,7 @@ namespace skyscraper5.Skyscraper.Scraper
{ {
MultiprotocolEncapsulationDecoder blockstreamDecoder = new MultiprotocolEncapsulationDecoder(this); MultiprotocolEncapsulationDecoder blockstreamDecoder = new MultiprotocolEncapsulationDecoder(this);
DvbContext.RegisterPacketProcessor(0x0020, new PsiDecoder(0x0020,blockstreamDecoder)); DvbContext.RegisterPacketProcessor(0x0020, new PsiDecoder(0x0020,blockstreamDecoder));
UiJunction?.NotifyBlockstreamCarrier(); UiJunction?.EnableUiFeature(SkyscraperUiFeature.BlockstreamAnalysis);
LogEvent(SkyscraperContextEvent.SpecialTsMode, "Blockstream Carrier TS detected."); LogEvent(SkyscraperContextEvent.SpecialTsMode, "Blockstream Carrier TS detected.");
SpecialTsType = 5; SpecialTsType = 5;
return; return;
@ -1436,6 +1445,8 @@ namespace skyscraper5.Skyscraper.Scraper
public void UpdateNotification(UpdateNotificationGroup common, UpdateNotificationTarget target, ushort ProgramNumber) public void UpdateNotification(UpdateNotificationGroup common, UpdateNotificationTarget target, ushort ProgramNumber)
{ {
UiJunction?.OnSsuNotification(common, target, ProgramNumber);
/*int hashCode = target.GetHashCode(); /*int hashCode = target.GetHashCode();
if (!ScraperStorage.TestForUpdateNotification(hashCode, common)) if (!ScraperStorage.TestForUpdateNotification(hashCode, common))
{ {
@ -1567,9 +1578,7 @@ namespace skyscraper5.Skyscraper.Scraper
if (isObjectCarousel) if (isObjectCarousel)
throw new NotImplementedException(); throw new NotImplementedException();
UiJunction?.SetMemorySaverMode(true);
ObjectStorage.DataCarouselModuleArrival(CurrentNetworkId.Value, CurrentTransportStreamId.Value, elementaryPid, moduleModuleId, moduleModuleVersion, result); ObjectStorage.DataCarouselModuleArrival(CurrentNetworkId.Value, CurrentTransportStreamId.Value, elementaryPid, moduleModuleId, moduleModuleVersion, result);
UiJunction?.SetMemorySaverMode(false);
LogEvent(SkyscraperContextEvent.ModuleDownloadComplete, String.Format("Module {0}, Version {1}", moduleModuleId, moduleModuleVersion)); LogEvent(SkyscraperContextEvent.ModuleDownloadComplete, String.Format("Module {0}, Version {1}", moduleModuleId, moduleModuleVersion));
@ -1717,6 +1726,7 @@ namespace skyscraper5.Skyscraper.Scraper
private IpTrafficHandler ipTrafficHandler; private IpTrafficHandler ipTrafficHandler;
public void OnIpDatagram(int pid, byte[] payload) public void OnIpDatagram(int pid, byte[] payload)
{ {
UiJunction?.EnableUiFeature(SkyscraperUiFeature.IpTrafficAnalysis);
if (ipTrafficHandler == null) if (ipTrafficHandler == null)
{ {
StorageConnectionManager storageConnectionManager = StorageConnectionManager.GetInstance(); StorageConnectionManager storageConnectionManager = StorageConnectionManager.GetInstance();
@ -1832,7 +1842,7 @@ namespace skyscraper5.Skyscraper.Scraper
{ {
LogEvent(SkyscraperContextEvent.LearnDns, String.Format("{0} = {1}", DnsCache.LastLearned.Value, DnsCache.LastLearned.Key)); LogEvent(SkyscraperContextEvent.LearnDns, String.Format("{0} = {1}", DnsCache.LastLearned.Value, DnsCache.LastLearned.Key));
} }
UiJunction?.NotifyMpeTraffic(iti, ipv4Packet.Length); UiJunction?.NotifyMpeTraffic(iti, ipv4Packet);
if (trafficInfos.Add(iti)) if (trafficInfos.Add(iti))
{ {
@ -1993,6 +2003,7 @@ namespace skyscraper5.Skyscraper.Scraper
child.InitalizeFilterChain(packetFilters.ToArray()); child.InitalizeFilterChain(packetFilters.ToArray());
subSkyscrapers.Add(identifier, child); subSkyscrapers.Add(identifier, child);
UiJunction?.OnDetectionOfInnerTs(child, identifier);
} }
SkyscraperContext context = subSkyscrapers[identifier]; SkyscraperContext context = subSkyscrapers[identifier];
@ -2185,6 +2196,8 @@ namespace skyscraper5.Skyscraper.Scraper
public void OnIpMacNotification(int sourcePid, Dvb.DataBroadcasting.IntModel.Platform platform, Target target, Operational operational) public void OnIpMacNotification(int sourcePid, Dvb.DataBroadcasting.IntModel.Platform platform, Target target, Operational operational)
{ {
UiJunction?.OnIpMacNotification(sourcePid, platform, target, operational);
IEnumerable<IpMacNotification> ipMacNotifications = IpMacNotification.FlatMap(platform, target, operational); IEnumerable<IpMacNotification> ipMacNotifications = IpMacNotification.FlatMap(platform, target, operational);
foreach (IpMacNotification notification in ipMacNotifications) foreach (IpMacNotification notification in ipMacNotifications)
{ {

16
skyscraper8/Skyscraper/Scraper/SkyscraperUiFeature.cs Normal file
View File

@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace skyscraper8.Skyscraper.Scraper
{
public enum SkyscraperUiFeature
{
BbframeAnalysis,
DocsisAnalysis,
BlockstreamAnalysis,
IpTrafficAnalysis
}
}