ft/skyscraper8
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Merge remote-tracking branch 'origin/bbframes-revisited'
All checks were successful
🚀 Pack skyscraper8 / make-zip (push) Successful in 1m17s
Details

Browse Source 
# Conflicts:
#	skyscraper8/GS/BbframeDeencapsulator2.cs
#	skyscraper8/GS/MisProcessor.cs
#	skyscraper8/GS/MisProcessors/0x00_Gse.cs
#	skyscraper8/GS/MisProcessors/UnknownStreamType.cs
#	skyscraper8/GS/Stid135TestingProgram.cs
#	skyscraper8/Program.cs
This commit is contained in:
feyris-tan 2025-10-16 08:54:10 +02:00
commit 41d90d9360
23 changed files with 209 additions and 1045 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
skyscraper8/GS/BBHeader.cs Normal file
View File

@ -0,0 +1,82 @@
using skyscraper5.Skyscraper;
namespace skyscraper8.GSE;
public class BBHeader : Validatable
{
public BBHeader(byte[] bbframe, int offset)
{
ReadOnlySpan<byte> BbHeaderSpan = new ReadOnlySpan<byte>(bbframe, offset, 10);
//byte 1, MATYPE-1
TsGs = (BbHeaderSpan[0] & 0xc0) >> 6;
SisMis = (BbHeaderSpan[0] & 0x20) != 0; //1 single, 0 multi
CcmAcm = (BbHeaderSpan[0] & 0x10) != 0; //1 ccm, 0 acm
Issyi = (BbHeaderSpan[0] & 0x08) != 0; //
Npd = (BbHeaderSpan[0] & 0x04) != 0;
Ro = (BbHeaderSpan[0] & 0x03);
Matype2 = (BbHeaderSpan[1]);
UserPacketLength = BbHeaderSpan[2] << 8 | BbHeaderSpan[3];
UserPacketLength /= 8;
DataFieldLength = BbHeaderSpan[4] << 8 | BbHeaderSpan[5];
DataFieldLength /= 8;
SyncByte = BbHeaderSpan[6];
SyncD = BbHeaderSpan[7] << 8 | BbHeaderSpan[8];
ChecksumValid = DvbCrc8.Compute(BbHeaderSpan) == 0;
Valid = ChecksumValid;
}
public bool ChecksumValid { get; private set; }
public int SyncD { get; private set; }
public byte SyncByte { get; private set; }
public int DataFieldLength { get; private set; }
public int UserPacketLength { get; private set; }
/// <summary>
/// If MIS, this is the Input Stream Identifier
/// </summary>
public byte Matype2 { get; private set; }
/// <summary>
/// Transmission Roll-off factor
/// </summary>
public int Ro { get; private set; }
/// <summary>
/// Null Packet Deletion
/// </summary>
public bool Npd { get; private set; }
/// <summary>
/// If true, the ISSY field is inserted after UP
/// </summary>
public bool Issyi { get; set; }
/// <summary>
/// True if CCM, False if ACM
/// </summary>
public bool CcmAcm { get; private set; }
/// <summary>
/// True if SIS, False if MIS
/// </summary>
public bool SisMis { get; private set; }
/// <summary>
/// 0 = GS, packetized
/// 1 = GS, continuous
/// 2 = GSE-HEM
/// 3 = Transport
/// </summary>
public int TsGs { get; private set; }
}

40
skyscraper8/GS/BBframeDeencapsulator3.cs Normal file
View File

@ -0,0 +1,40 @@
using log4net;
namespace skyscraper8.GSE;
public class BbframeDeencapsulator3 : IBbframeDeencapsulator
{
private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
private long numPushed;
public void PushPacket(byte[] bbframe)
{
//byte 0, sync byte
if (bbframe[0] != 0xb8)
{
if (numPushed == 0)
{
logger.InfoFormat("The stream started in the middle of a BBFrame, let's skip to the start of the next one.");
}
return;
}
numPushed++;
BBHeader bbHeader = new BBHeader(bbframe, 1);
if (!bbHeader.ChecksumValid)
return;
if (!bbHeader.Valid)
return;
if (mis == null)
mis = new IMisHandler[256];
if (mis[bbHeader.Matype2] == null)
{
logger.InfoFormat("Found a stream on MIS {0}",bbHeader.Matype2);
mis[bbHeader.Matype2] = new GsTypeDetector();
}
mis[bbHeader.Matype2].PushFrame(bbHeader, new ReadOnlySpan<byte>(bbframe, 11, bbframe.Length - 11));
}
private IMisHandler[] mis;
}

0
skyscraper8/GSE/BbframeDumper.cs → skyscraper8/GS/BbframeDumper.cs
View File

26
skyscraper8/GS/Crc8.cs Normal file
View File

@ -0,0 +1,26 @@
namespace skyscraper8.GSE;
public static class DvbCrc8
{
private const byte Polynomial = 0xD5; // x^8 + x^7 + x^6 + x^4 + x^2 + 1
public static byte Compute(ReadOnlySpan<byte> data)
{
byte crc = 0x00;
foreach (byte b in data)
{
crc ^= b; // XOR byte into top of crc register
for (int i = 0; i < 8; i++)
{
if ((crc & 0x80) != 0)
crc = (byte)((crc << 1) ^ Polynomial);
else
crc <<= 1;
}
}
return crc;
}
}

9
skyscraper8/GS/GsTypeDetector.cs Normal file
View File

@ -0,0 +1,9 @@
namespace skyscraper8.GSE;
public class GsTypeDetector : IMisHandler
{
public void PushFrame(BBHeader bbHeader, ReadOnlySpan<byte> readOnlySpan)
{
throw new NotImplementedException();
}
}

0
skyscraper8/GSE/IBbframeDeencapsulator.cs → skyscraper8/GS/IBbframeDeencapsulator.cs
View File

6
skyscraper8/GS/IMisHandler.cs Normal file
View File

@ -0,0 +1,6 @@
namespace skyscraper8.GSE;
public interface IMisHandler
{
void PushFrame(BBHeader bbHeader, ReadOnlySpan<byte> readOnlySpan);
}

0
skyscraper8/GSE/BbfUdpDecap.cs → skyscraper8/GS/POC/BbfUdpDecap.cs
View File

14
skyscraper8/GSE/Pts2Bbf.cs → skyscraper8/GS/POC/Pts2Bbf.cs
View File

@ -11,7 +11,7 @@ public class Pts2Bbf
{
private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
public static void Run(FileInfo file, bool useUdpDecap = false)
public static void Run(FileInfo file, bool useUdpDecap = false, IPEndPoint ipEndPoint = null)
{
if (!file.Exists)
{
@ -24,10 +24,14 @@ public class Pts2Bbf
{
string changeExtension = Path.ChangeExtension(file.FullName, ".sdecap");
BbfUdpDecap.UdpDecapFileOutput udpDecapSink = new BbfUdpDecap.UdpDecapFileOutput(new FileInfo(changeExtension));
if (ipEndPoint == null)
{
ipEndPoint = new IPEndPoint(IPAddress.Parse("239.199.2.1"), 1234);
}
BbfUdpDecap bbfUdpDecap = new BbfUdpDecap();
bbfUdpDecap.SetTargetPort(1234);
bbfUdpDecap.SetTargetIp(IPAddress.Parse("239.199.2.1"));
bbfUdpDecap.SetTargetPort(ipEndPoint.Port);
bbfUdpDecap.SetTargetIp(ipEndPoint.Address);
bbfUdpDecap.Sink = udpDecapSink;
dumper = bbfUdpDecap;
}
@ -40,7 +44,7 @@ public class Pts2Bbf
FileStream fileStream = file.OpenRead();
TsContext mpeg2 = new TsContext();
mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(new NullGsEventHandler(),dumper));
mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(dumper));
DataStorage dataStorage = new InMemoryScraperStorage();
ObjectStorage objectStorage = new NullObjectStorage();
SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, dataStorage, objectStorage);

29
skyscraper8/GS/POC/Stid135Test.cs Normal file
View File

@ -0,0 +1,29 @@
using log4net;
using skyscraper5.Mpeg2;
using skyscraper5.Skyscraper.Scraper;
using skyscraper5.Skyscraper.Scraper.Storage.InMemory;
using skyscraper8.Skyscraper.Scraper.Storage;
namespace skyscraper8.GSE;
public class Stid135Test
{
private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
public static void Run(FileInfo file)
{
FileStream fileStream = file.OpenRead();
BbframeDeencapsulator3 decap = new BbframeDeencapsulator3();
TsContext mpeg2 = new TsContext();
mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(decap));
DataStorage dataStorage = new InMemoryScraperStorage();
ObjectStorage objectStorage = new NullObjectStorage();
SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, dataStorage, objectStorage);
skyscraper.InitalizeFilterChain();
skyscraper.IngestFromStream(fileStream);
fileStream.Close();
logger.Info("STiD135 Tester exiting");
}
}

21
skyscraper8/GSE/Stid135BbFrameReader.cs → skyscraper8/GS/Stid135BbFrameReader.cs
View File

@ -11,15 +11,13 @@ namespace skyscraper8.GSE
{
internal class Stid135BbFrameReader : ITsPacketProcessor
{
private IGsEventHandler mpeEventHandler;
private IBbframeDeencapsulator deencapsulator;
public Stid135BbFrameReader(IGsEventHandler mpeEventHandler, IBbframeDeencapsulator deencapsulator = null)
public Stid135BbFrameReader(IBbframeDeencapsulator deencapsulator = null)
{
if (deencapsulator == null)
deencapsulator = new BbframeDeencapsulator();
deencapsulator = new BbframeDeencapsulator3();
this.mpeEventHandler = mpeEventHandler;
this.deencapsulator = deencapsulator;
}
@ -56,21 +54,6 @@ namespace skyscraper8.GSE
{
outbuf.Write(packets, 9, packets[7] - 1);
}
/*if ((packets[8] & 0xff) == 0xb8)
{
if (outbuf != null)
{
byte[] chi = outbuf.ToArray();
deencapsulator.PushPacket(chi);
}
outbuf = new MemoryStream();
outbuf.Write(packets,9, packets[7] - 1);
}
else
{
if (outbuf != null)
outbuf.Write(packets, 9, packets[7] - 1);
}*/
}
}
}

446
skyscraper8/GSE/BbframeDeencapsulator.cs
View File

@ -1,446 +0,0 @@
using skyscraper5.Mpeg2;
using skyscraper5.Skyscraper;
using skyscraper5.Skyscraper.IO;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.NetworkInformation;
using System.Net.Sockets;
using System.Runtime.Serialization;
using System.Text;
using System.Threading.Tasks;
using log4net;
namespace skyscraper8.GSE
{
internal class BbframeDeencapsulator : IBbframeDeencapsulator
{
private bool interruptedGseHem;
private MemoryStream interruptedGseHemBuffer;
private bool interruptedGseHemWantsCrc32;
public IGsEventHandler MpeEventHandler { get; set; }
private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
private bool shownNonGseWarning;
public void PushPacket(byte[] bbframe)
{
MemoryStream ms = new MemoryStream(bbframe, false);
//BBHeader
byte matype1 = ms.ReadUInt8();
int tsGsField = (matype1 & 0xc0) >> 6;
bool sisMisField = (matype1 & 0x20) != 0;
bool ccmAcmField = (matype1 & 0x10) != 0;
bool issyi = (matype1 & 0x08) != 0;
bool npd = (matype1 & 0x04) != 0;
int ro = matype1 & 0x03;
byte matype2 = ms.ReadUInt8();
ushort userPacketLength = ms.ReadUInt16BE();
ushort dataFieldLength = ms.ReadUInt16BE();
byte sync = ms.ReadUInt8();
ushort syncd = ms.ReadUInt16BE();
byte crc8 = ms.ReadUInt8();
if (userPacketLength == 0 && dataFieldLength == 0)
return;
switch (tsGsField)
{
case 1:
if (sync != 0)
{
if (!shownNonGseWarning)
{
logger.WarnFormat("This stream is a valid GS, but also contains packets which are not GSE (type 0), but of type {0} . Please share a sample of this stream!", sync);
shownNonGseWarning = true;
}
return;
}
int bytes = dataFieldLength / 8;
if (ms.GetAvailableBytes() < bytes)
return;
HandleContinous(ms);
break;
case 2:
int hemBytes = dataFieldLength / 8;
if (ms.GetAvailableBytes() < hemBytes)
return;
HandleGseHem(ms);
break;
default: //0 = generic packetized, 1 = generic continouus, 2 = gse, 3 = ts
logger.Warn(string.Format("Unsupported: TS/GS field says 0x{0:X2}, please share a sample of this stream.", tsGsField));
break;
}
}
private GseFragmentation[] gseFragmentations;
private void HandleContinous(MemoryStream ms)
{
ms.Position = 10;
while (ms.GetAvailableBytes() > 0)
{
GsePacket packet = GsePacket.Read(ms);
if (packet.IsPadding)
{
break;
}
if (!packet.Valid)
{
break;
}
if (packet.IsCompletePacket)
{
if (ValidateEthertype(packet.ProtocolType.Value))
{
MpeEventHandler.OnIpDatagram(PID, packet.GseData);
}
else
{
logger.WarnFormat("Unknown EtherType: 0x{0:X4}", packet.ProtocolType.Value);
}
continue;
}
if (packet.StartIndicator && !packet.EndIndicator)
{
if (gseFragmentations == null)
gseFragmentations = new GseFragmentation[256];
gseFragmentations[packet.FragmentId.Value] = new GseFragmentation();
gseFragmentations[packet.FragmentId.Value].AddPacket(packet);
continue;
}
if (!packet.StartIndicator && packet.EndIndicator)
{
if (gseFragmentations == null)
continue;
if (gseFragmentations[packet.FragmentId.Value] == null)
continue;
gseFragmentations[packet.FragmentId.Value].AddPacket(packet);
continue;
}
if (!packet.StartIndicator && !packet.EndIndicator)
{
if (gseFragmentations == null)
continue;
throw new NotImplementedException();
}
throw new NotImplementedException();
}
}
private bool gseNeedMore;
private MemoryStream gseAssembler;
public int PID;
private void HandleGseHem(MemoryStream ms)
{
ms.Position = 10;
byte syncByte = ms.ReadUInt8();
ms.Position = 10;
long availableBytes = ms.GetAvailableBytes();
byte[] readBytes = ms.ReadBytes(availableBytes);
GseAssemblyState assemblyState = ValidateGse(gseAssembler, true);
if (assemblyState == GseAssemblyState.NOT_YET_BEGUN)
{
if ((syncByte & 0xc0) == 0xc0)
{
//Aktuell ist der GSE Assembler nicht beschäftigt, aber wir haben ein Sync-Byte, also kann er loslegen.
gseAssembler = new MemoryStream();
gseAssembler.Write(readBytes, 0, readBytes.Length);
return;
}
else
{
//Aktuell ist der GSE Assembler nicht beschäftigt, und wir haben kein gültiges Sync-Byte, also weg damit.
//Console.WriteLine("gse not in sync yet. sync byte = {0}", syncByte);
return;
}
}
else if (assemblyState == GseAssemblyState.NEED_MORE_DATA)
{
gseAssembler.Write(readBytes, 0, readBytes.Length);
return;
}
else if (assemblyState == GseAssemblyState.VALID)
{
gseAssembler.Position = 0;
AssembleGse(gseAssembler);
//Console.WriteLine("assembled {0} next sync byte is {1}", sucessfulAssembles++,syncByte);
if (sucessfulAssembles == 181)
{
}
gseAssembler = null;
HandleGseHem(ms);
return;
}
else if (assemblyState == GseAssemblyState.BROKEN_SUSPECT_ETHERTYPE)
{
gseAssembler = null;
HandleGseHem(ms);
return;
}
else if (assemblyState == GseAssemblyState.BROKEN_NEGATIVE_LENGTH)
{
gseAssembler = null;
HandleGseHem(ms);
return;
}
else
{
throw new NotImplementedException(string.Format("Unknown GSE assembler state: {0}, sync byte 0x{1}", assemblyState, syncByte));
}
}
private long sucessfulAssembles;
enum GseAssemblyState
{
NOT_YET_BEGUN,
VALID,
BROKEN_NEGATIVE_LENGTH,
NEED_MORE_DATA,
BROKEN_SUSPECT_LENGTH,
BROKEN_SUSPECT_ETHERTYPE,
VALID_NULL_PACKET, //somehow these only show up on TBS6903x, not DD Max SX8?
}
private GseAssemblyState ValidateGse(MemoryStream ms, bool tryFix = false)
{
if (ms == null)
return GseAssemblyState.NOT_YET_BEGUN;
long backupPosition = ms.Position;
ms.Position = 0;
while (ms.GetAvailableBytes() > 2)
{
long fixPosition = ms.Position;
//GSE-Header
byte byteA = ms.ReadUInt8();
bool startIndicator = (byteA & 0x80) != 0;
bool endIndicator = (byteA & 0x40) != 0;
int labelIndicator = (byteA & 0x30) >> 4;
if (!startIndicator && !endIndicator && labelIndicator == 0)
{
//end of base band frame
ms.Position = backupPosition;
return GseAssemblyState.VALID;
}
else
{
byte byteB = ms.ReadUInt8();
int gseLength = byteA & 0x0f;
gseLength <<= 8;
gseLength += byteB;
//Console.WriteLine("GSE Length: {0}", gseLength);
if (gseLength > 9000)
{
if (tryFix)
{
ms.Position = fixPosition;
ms.WriteUInt8Repeat(0, (int)ms.GetAvailableBytes());
break;
}
ms.Position = backupPosition;
return GseAssemblyState.BROKEN_SUSPECT_LENGTH;
}
if (!startIndicator || !endIndicator)
{
byte fragId = ms.ReadUInt8();
gseLength--;
}
if (startIndicator && !endIndicator)
{
if (2 > ms.GetAvailableBytes())
{
ms.Position = backupPosition;
return GseAssemblyState.NEED_MORE_DATA;
}
ushort totalLength = ms.ReadUInt16BE();
gseLength -= 2;
}
if (startIndicator)
{
if (2 > ms.GetAvailableBytes())
{
ms.Position = backupPosition;
return GseAssemblyState.NEED_MORE_DATA;
}
ushort protocolType = ms.ReadUInt16BE();
if (!ValidateEthertype(protocolType))
{
if (tryFix)
{
ms.Position = fixPosition;
ms.WriteUInt8Repeat(0, (int)ms.GetAvailableBytes());
break;
}
ms.Position = backupPosition;
return GseAssemblyState.BROKEN_SUSPECT_ETHERTYPE;
}
gseLength -= 2;
if (labelIndicator == 0)
{
if (6 > ms.GetAvailableBytes())
{
ms.Position = backupPosition;
return GseAssemblyState.NEED_MORE_DATA;
}
PhysicalAddress sixByteLabel = new PhysicalAddress(ms.ReadBytes(6));
gseLength -= 6;
}
else if (labelIndicator == 1)
{
byte[] threeByteLabel = ms.ReadBytes(3);
gseLength -= 3;
}
}
if (!startIndicator && endIndicator)
gseLength -= 4;
int startCrc32 = (int)ms.Position;
if (gseLength < 0)
{
if (tryFix)
{
ms.Position = fixPosition;
ms.WriteUInt8Repeat(0, (int)ms.GetAvailableBytes());
break;
}
ms.Position = backupPosition;
return GseAssemblyState.BROKEN_NEGATIVE_LENGTH;
}
if (gseLength > ms.GetAvailableBytes())
{
ms.Position = backupPosition;
return GseAssemblyState.NEED_MORE_DATA;
}
ms.Position += gseLength;
if (!startIndicator && endIndicator)
{
if (4 > ms.GetAvailableBytes())
{
ms.Position = backupPosition;
return GseAssemblyState.NEED_MORE_DATA;
}
uint crc32 = ms.ReadUInt32BE();
int endCrc32 = (int)ms.Position;
DvbCrc32.ValidateCrc(ms, startCrc32, endCrc32);
}
}
}
ms.Position = backupPosition;
return GseAssemblyState.VALID;
}
private bool ValidateEthertype(ushort ethertype)
{
if (ethertype == 0x0800)
return true; //IPv4
if (ethertype == 0x86DD)
return true; //IPv6
else
return false; //There are many more valid values, but since we don't speak anything aside from IPv4/IPv6, we discard it.
}
private void AssembleGse(MemoryStream ms)
{
ushort protocolType = 0;
while (ms.GetAvailableBytes() > 2)
{
//GSE-Header
byte byteA = ms.ReadUInt8();
bool startIndicator = (byteA & 0x80) != 0;
bool endIndicator = (byteA & 0x40) != 0;
int labelIndicator = (byteA & 0x30) >> 4;
if (!startIndicator && !endIndicator && labelIndicator == 0)
{
//end of base band frame
return;
}
else
{
byte byteB = ms.ReadUInt8();
int gseLength = byteA & 0x0f;
gseLength <<= 8;
gseLength += byteB;
if (!startIndicator || !endIndicator)
{
byte fragId = ms.ReadUInt8();
gseLength--;
}
if (startIndicator && !endIndicator)
{
ushort totalLength = ms.ReadUInt16BE();
gseLength -= 2;
}
if (startIndicator)
{
protocolType = ms.ReadUInt16BE();
gseLength -= 2;
if (labelIndicator == 0)
{
PhysicalAddress sixByteLabel = new PhysicalAddress(ms.ReadBytes(6));
gseLength -= 6;
}
else if (labelIndicator == 1)
{
byte[] threeByteLabel = ms.ReadBytes(3);
gseLength -= 3;
}
}
if (!startIndicator && endIndicator)
gseLength -= 4;
int startCrc32 = (int)ms.Position;
byte[] payload = ms.ReadBytes(gseLength);
if (!startIndicator && endIndicator)
{
uint crc32 = ms.ReadUInt32BE();
int endCrc32 = (int)ms.Position;
DvbCrc32.ValidateCrc(ms, startCrc32, endCrc32);
}
switch (protocolType)
{
case 0x0800: //IPv4
MpeEventHandler.OnIpDatagram(PID, payload);
break;
default:
//throw new NotImplementedException(String.Format("Unknown GSE Protocol ID 0x{0:X4}", protocolType));
break;
}
}
}
}
}
}

124
skyscraper8/GSE/BbframeDeencapsulator2.cs
View File

@ -1,124 +0,0 @@
using log4net;
using skyscraper5.Skyscraper.IO;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
using System.Xml.Linq;
using skyscraper5.Skyscraper.Net.Pcap;
using skyscraper8.GSE.MisProcessors;
namespace skyscraper8.GSE
{
public class BbframeDeencapsulator2 : IBbframeDeencapsulator
{
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public struct bbheader
{
public byte MaType1;
public byte MaType2;
public byte Upl1;
public byte Upl2;
public byte Dfl1;
public byte Dfl2;
public byte Sync;
public byte SyncD1;
public byte SyncD2;
public byte Crc8;
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public unsafe struct Layer3
{
public byte l3sync;
public bbheader header;
public fixed byte Data[7254 - 10];
}
public long NumPushed { get; private set; }
private byte[] inData;
private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
public unsafe void PushPacket(byte[] bbframe)
{
NumPushed++;
if (inData == null)
{
inData = new byte[8100];
}
MemoryStream inbuf = new MemoryStream(bbframe);
Layer3 myLayer;
while (true)
{
if (inbuf.Position == inbuf.Length)
return;
if (inbuf.Peek() == 0xb8)
break;
inbuf.ReadByte();
}
while (true)
{
int fail = inbuf.Read(inData, 0, 11);
if (fail != 11)
break;
myLayer = MemoryMarshal.Read<Layer3>(inData);
if (myLayer.l3sync != 0xb8)
{
while (true)
{
if (inbuf.Position == inbuf.Length)
return;
if (inbuf.Peek() == 0xb8)
break;
fail = inbuf.Read(inData, 0, 1);
if (fail != 1)
break;
}
}
int bblength = myLayer.header.Dfl1 << 8 | myLayer.header.Dfl2;
bblength >>= 3;
fail = inbuf.Read(inData, 11, bblength);
if (fail != bblength)
break;
myLayer = MemoryMarshal.Read<Layer3>(inData);
byte streamType = myLayer.header.Sync;
byte mis = myLayer.header.MaType2;
if (misStreams == null)
misStreams = new MisProcessor[256];
if (misStreams[mis] == null)
{
logger.InfoFormat("Found MIS {0}", mis);
MisProcessor processor;
switch (streamType)
{
case 0:
processor = new _0x00_Gse();
break;
default:
processor = new UnknownStreamType(streamType,mis);
break;
}
misStreams[mis] = processor;
}
misStreams[mis].PushPacket(bblength, myLayer);
}
}
private MisProcessor[] misStreams;
}
}

14
skyscraper8/GSE/GsEventHandler.cs
View File

@ -1,14 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace skyscraper5.Skyscraper
{
internal interface IGsEventHandler
{
void OnIpDatagram(int pid, byte[] payload);
void GsIpTrafficDetected();
}
}

26
skyscraper8/GSE/GseFragmentation.cs
View File

@ -1,26 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace skyscraper8.GSE
{
internal class GseFragmentation
{
public void AddPacket(GsePacket packet)
{
if (spanningGsePackets == null)
{
spanningGsePackets = new List<GsePacket>();
TotalLength = packet.TotalLength.Value;
}
spanningGsePackets.Add(packet);
}
public ushort TotalLength { get; private set; }
private List<GsePacket> spanningGsePackets;
}
}

167
skyscraper8/GSE/GsePacket.cs
View File

@ -1,167 +0,0 @@
using skyscraper5.Skyscraper.IO;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.NetworkInformation;
using System.Text;
using System.Threading.Tasks;
using skyscraper5.Skyscraper;
namespace skyscraper8.GSE
{
internal class GsePacket : Validatable
{
public static GsePacket Read(MemoryStream ms)
{
GsePacket packet = new GsePacket();
byte byteA = ms.ReadUInt8();
packet.StartIndicator = (byteA & 0x80) != 0;
packet.EndIndicator = (byteA & 0x40) != 0;
packet.LabelIndicator = (byteA & 0x30) >> 4;
if (!packet.StartIndicator && !packet.EndIndicator && packet.LabelIndicator == 0)
{
//End of baseband frame
return packet;
}
else
{
byte byteB = ms.ReadUInt8();
int gseLength = byteA & 0x0f;
gseLength <<= 8;
gseLength += byteB;
if (!packet.StartIndicator || !packet.EndIndicator)
{
packet.FragmentId = ms.ReadUInt8();
gseLength--;
}
if (packet.StartIndicator && !packet.EndIndicator)
{
packet.TotalLength = ms.ReadUInt16BE();
gseLength -= 2;
}
if (packet.StartIndicator)
{
packet.ProtocolType = ms.ReadUInt16BE();
gseLength -= 2;
if (packet.LabelIndicator == 0)
{
packet.Label = new SixByteGseLabel(ms.ReadBytes(6));
gseLength -= 6;
}
else if (packet.LabelIndicator == 1)
{
packet.Label = new ThreeByteGseLabel(ms.ReadBytes(3));
gseLength -= 3;
}
}
if (!packet.StartIndicator && packet.EndIndicator)
gseLength -= 4;
if (gseLength < 0)
{
packet.Valid = false;
return packet;
}
if (gseLength > ms.GetAvailableBytes())
{
packet.Valid = false;
return packet;
}
packet.GseData = ms.ReadBytes(gseLength);
if (!packet.StartIndicator && packet.EndIndicator)
packet.Crc32 = ms.ReadUInt32BE();
}
packet.Valid = true;
return packet;
}
public uint? Crc32 { get; set; }
public byte[] GseData { get; set; }
public GseLabel Label { get; set; }
public ushort? ProtocolType { get; set; }
public ushort? TotalLength { get; set; }
public byte? FragmentId { get; set; }
public int LabelIndicator { get; private set; }
public bool EndIndicator { get; private set; }
public bool StartIndicator { get; private set; }
public bool IsPadding
{
get
{
return !StartIndicator && !EndIndicator && LabelIndicator == 0;
}
}
public bool IsCompletePacket
{
get
{
return StartIndicator && EndIndicator;
}
}
}
public abstract class GseLabel : Validatable
{
public abstract int LabelLength { get; }
public abstract string ToString();
}
public class ThreeByteGseLabel : GseLabel
{
public ThreeByteGseLabel(byte[] buffer)
{
if (buffer.Length != 3)
{
Valid = false;
return;
}
LabelBytes = buffer;
Valid = true;
}
public byte[] LabelBytes { get; private set; }
public override string ToString()
{
return BitConverter.ToString(LabelBytes);
}
public override int LabelLength => 3;
}
public class SixByteGseLabel : GseLabel
{
public SixByteGseLabel(byte[] buffer)
{
if (buffer.Length != 6)
{
Valid = false;
return;
}
MacAddress = new PhysicalAddress(buffer);
Valid = true;
}
public PhysicalAddress MacAddress { get; private set; }
public override string ToString()
{
return MacAddress.ToString();
}
public override int LabelLength => 6;
}
}

13
skyscraper8/GSE/MisProcessor.cs
View File

@ -1,13 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace skyscraper8.GSE
{
public interface MisProcessor
{
public unsafe void PushPacket(int bblength, BbframeDeencapsulator2.Layer3 myLayer);
}
}

131
skyscraper8/GSE/MisProcessors/0x00_Gse.cs
View File

@ -1,131 +0,0 @@
using skyscraper5.Skyscraper.Net.Pcap;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace skyscraper8.GSE.MisProcessors
{
internal class _0x00_Gse : MisProcessor
{
private byte[][] fragmentor;
private int[] fragmentorLength;
private int[] fragmentorPos;
private PcapWriter pcapWriter;
public _0x00_Gse()
{
fragmentor = new byte[256][];
fragmentorLength = new int[256];
fragmentorPos = new int[256];
FileStream fileStream = File.OpenWrite(string.Format("{0}.pcap", DateTime.Now.Ticks));
pcapWriter = new PcapWriter(fileStream, TcpdumpNetworkType.RawIp);
}
public unsafe void PushPacket(int bblength, BbframeDeencapsulator2.Layer3 myLayer)
{
int pos = 0;
while (pos < bblength - 4)
{
int gseLength = ((myLayer.Data[pos] & 0x0f) << 8) | (myLayer.Data[pos + 1]);
if ((myLayer.Data[pos] & 0xf0) == 0) break;
if (gseLength + 2 > bblength - pos) break;
if (!ProcessBbFrame(&myLayer.Data[pos], gseLength))
break;
pos += gseLength += 2;
}
}
private unsafe void Copy(byte* src, byte[] dst, int srcOffset, int dstOffset, int len)
{
for (int i = 0; i < len; i++)
{
dst[dstOffset + i] = src[srcOffset + i];
}
}
private unsafe bool ProcessBbFrame(byte* payload, int gseLength)
{
uint offset = 0;
uint fragId = 0;
if ((payload[0] & 0xc0) == 0x80)
{
fragId = payload[2];
int length = (payload[3] << 8) | payload[4];
if (fragmentor[fragId] != null)
fragmentor[fragId] = null;
fragmentor[fragId] = new byte[length + 2];
fragmentorLength[fragId] = length + 2;
fragmentor[fragId][0] = payload[0];
fragmentor[fragId][1] = payload[1];
fragmentor[fragId][0] |= 0xc0;
Copy(payload, fragmentor[fragId], 5, 2, gseLength - 3);
fragmentorPos[fragId] = gseLength - 1;
}
else if ((payload[0] & 0xc0) == 0x00)
{
fragId = payload[2];
if (fragmentor[fragId] == null)
return true;
Copy(payload, fragmentor[fragId], 3, fragmentorPos[fragId], gseLength - 1);
fragmentorPos[fragId] += gseLength - 1;
}
else if ((payload[0] & 0xc0) == 0x40)
{
fragId = payload[2];
if (fragmentor[fragId] == null)
return true;
Copy(payload, fragmentor[fragId], 3, fragmentorPos[fragId], gseLength - 5);
fragmentorPos[fragId] += gseLength - 1;
fixed (byte* p = fragmentor[fragId])
{
ProcessBbFrame(p, fragmentorLength[fragId]);
}
fragmentor[fragId] = null;
}
else if ((payload[0] & 0xc0) == 0xc0)
{
if (payload[offset + 2] == 0x00 && payload[offset + 3] == 0x04)
{
throw new NotImplementedException("ethertype 0x0400");
}
else if (payload[offset + 2] == 0x00 && payload[offset + 3] == 0x00)
{
throw new NotImplementedException("ethertype 0x0000");
}
else if (payload[offset + 2] == 0x08 && payload[offset + 3] == 0x00)
{
if ((payload[0] & 0x30) == 0x01)
{
offset += 3; //3-byte label
}
else if ((payload[0] & 0x30) == 0x00)
{
offset += 6; //MAC address
}
offset += 4;
//at this point, the IP header starts at "offset"
long sinkSize = gseLength + 2 - offset;
byte[] sink = new byte[sinkSize];
Copy(payload, sink, (int)offset, 0, (int)sinkSize);
pcapWriter.WritePacket(sink);
return true;
}
//throw new NotImplementedException("complete packet");
}
else if ((payload[0] & 0x0f0) == 0x00)
{
throw new NotImplementedException("padding packet");
}
return true;
}
}
}

38
skyscraper8/GSE/MisProcessors/UnknownStreamType.cs
View File

@ -1,38 +0,0 @@
using log4net;
using skyscraper5.Skyscraper.Scraper;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace skyscraper8.GSE.MisProcessors
{
internal class UnknownStreamType : MisProcessor
{
private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
private bool[] warningsSpoken;
private byte mis;
public UnknownStreamType(byte streamType, byte mis)
{
this.mis = mis;
this.warningsSpoken = new bool[256];
this.EncounterPacket(streamType);
}
private void EncounterPacket(byte streamType)
{
if (!warningsSpoken[streamType])
{
logger.WarnFormat("This is a valid GS, however, it not GSE (type 0), but type {0}, which is not supported yet. Please consider sharing a sample of this stream.", streamType);
warningsSpoken[streamType] = true;
}
}
public unsafe void PushPacket(int bblength, BbframeDeencapsulator2.Layer3 myLayer)
{
EncounterPacket(myLayer.header.Sync);
}
}
}

16
skyscraper8/GSE/NullGsEventHandler.cs
View File

@ -1,16 +0,0 @@
using skyscraper5.Skyscraper;
namespace skyscraper8.GSE;
public class NullGsEventHandler : IGsEventHandler
{
public void OnIpDatagram(int pid, byte[] payload)
{
}
public void GsIpTrafficDetected()
{
}
}

40
skyscraper8/GSE/Stid135TestingProgram.cs
View File

@ -1,40 +0,0 @@
using log4net;
using log4net.Repository.Hierarchy;
using skyscraper5.Mpeg2;
using skyscraper5.Skyscraper.Scraper;
using skyscraper5.Skyscraper.Scraper.Storage.InMemory;
using skyscraper8.Skyscraper.Scraper.Storage;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace skyscraper8.GSE
{
internal class Stid135TestingProgram
{
private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
public static void Run(FileInfo file)
{
if (!file.Exists)
{
logger.Error("File not found: " + file.FullName);
return;
}
FileStream fileStream = file.OpenRead();
TsContext mpeg2 = new TsContext();
mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(new NullGsEventHandler(),new BbframeDeencapsulator2()));
DataStorage dataStorage = new InMemoryScraperStorage();
ObjectStorage objectStorage = new NullObjectStorage();
SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, dataStorage, objectStorage);
skyscraper.InitalizeFilterChain();
skyscraper.IngestFromStream(fileStream);
fileStream.Close();
logger.Info("Stid135TestingProgram finished");
}
}
}

6
skyscraper8/Program.cs
View File

@ -326,9 +326,9 @@ namespace skyscraper5
if (args[0].ToLowerInvariant().Equals("stid135test"))
{
FileInfo fi = new FileInfo(args[1]);
Stid135TestingProgram.Run(fi);
return;
FileInfo fi = new FileInfo(args[1]);
Stid135Test.Run(fi);
return;
}
}

6
skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs
View File

@ -92,7 +92,7 @@ namespace skyscraper5.Skyscraper.Scraper
IEitEventHandler, IAitEventHandler, ISubtitleEventHandler,
UpdateNotificationEventHandler, DataCarouselEventHandler, RdsEventHandler, IScte35EventHandler,
IAutodetectionEventHandler, IRstEventHandler, IRntEventHandler, IMultiprotocolEncapsulationEventHandler, ObjectCarouselEventHandler, T2MIEventHandler,
IDisposable, IFrameGrabberEventHandler, IntEventHandler, IRctEventHandler, IGsEventHandler, ISkyscraperContext, IDocsisEventHandler, AbertisDecoderEventHandler, Id3Handler,
IDisposable, IFrameGrabberEventHandler, IntEventHandler, IRctEventHandler, ISkyscraperContext, IDocsisEventHandler, AbertisDecoderEventHandler, Id3Handler,
InteractionChannelHandler, SgtEventHandler, IDvbNipEventHandler, UleEventHandler, OtvSsuHandler, NdsSsuHandler
{
public const bool ALLOW_STREAM_TYPE_AUTODETECTION = true;
@ -301,7 +301,7 @@ namespace skyscraper5.Skyscraper.Scraper
{
if (!DvbContext.IsPidProcessorPresent(0x010e))
{
DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(this));
DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader());
UiJunction?.SetGseMode();
LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected.");
SpecialTsType = 3;
@ -356,7 +356,7 @@ namespace skyscraper5.Skyscraper.Scraper
{
if (!DvbContext.IsPidProcessorPresent(0x010e))
{
DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(this));
DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader());
UiJunction?.SetGseMode();
LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected.");
SpecialTsType = 3;