diff --git a/.gitea/workflows/demo.yaml b/.gitea/workflows/demo.yaml index 5ba17c5..1ea1dff 100644 --- a/.gitea/workflows/demo.yaml +++ b/.gitea/workflows/demo.yaml @@ -33,7 +33,8 @@ jobs: uses: akkuman/gitea-release-action@v1 with: name: ${{ gitea.sha }} - md5sum: true - sha256sum: true + md5sum: false + sha256sum: false + tag_name: release_tag files: |- ${{ gitea.workspace }}/skyscraper8-${{ gitea.sha }}.zip \ No newline at end of file diff --git a/skyscraper8.sln.DotSettings.user b/skyscraper8.sln.DotSettings.user index 33fbc3f..ff66e63 100644 --- a/skyscraper8.sln.DotSettings.user +++ b/skyscraper8.sln.DotSettings.user @@ -1,4 +1,7 @@  + ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded <data><HostParameters type="LocalHostParameters" /><Argument type="StandaloneArgument"><Arguments IsNull="False"></Arguments><FileName IsNull="False"></FileName><WorkingDirectory IsNull="False"></WorkingDirectory><Scope><ProcessFilters /></Scope></Argument><Info type="TimelineInfo" /><CoreOptions type="CoreOptions"><CoreTempPath IsNull="False"></CoreTempPath><RemoteEndPoint IsNull="False"></RemoteEndPoint><AdditionalEnvironmentVariables /></CoreOptions><HostOptions type="HostOptions"><HostTempPath IsNull="False"></HostTempPath></HostOptions></data> \ No newline at end of file diff --git a/skyscraper8/Dvb/DataBroadcasting/MultiprotocolEncapsulationEventHandler.cs b/skyscraper8/Dvb/DataBroadcasting/MultiprotocolEncapsulationEventHandler.cs index b99bdaf..09fb4cb 100644 --- a/skyscraper8/Dvb/DataBroadcasting/MultiprotocolEncapsulationEventHandler.cs +++ b/skyscraper8/Dvb/DataBroadcasting/MultiprotocolEncapsulationEventHandler.cs @@ -7,7 +7,7 @@ using skyscraper5.Ietf.Rfc971; namespace skyscraper5.Dvb.DataBroadcasting { - interface IMultiprotocolEncapsulationEventHandler + public interface IMultiprotocolEncapsulationEventHandler { void OnIpv4PacketArrival(InternetHeader internetHeader, byte[] ipv4Packet); void OnIpDatagram(int sourcePid, byte[] payload); diff --git a/skyscraper8/GS/BBframeDeencapsulator3.cs b/skyscraper8/GS/BBframeDeencapsulator3.cs index 791235c..33996fa 100644 --- a/skyscraper8/GS/BBframeDeencapsulator3.cs +++ b/skyscraper8/GS/BBframeDeencapsulator3.cs @@ -1,9 +1,16 @@ using log4net; +using skyscraper5.Dvb.DataBroadcasting; namespace skyscraper8.GSE; public class BbframeDeencapsulator3 : IBbframeDeencapsulator { + public BbframeDeencapsulator3(IMultiprotocolEncapsulationEventHandler mpeEventHandler) + { + _mpeEventHandler = mpeEventHandler; + } + + private readonly IMultiprotocolEncapsulationEventHandler _mpeEventHandler; private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); private long numPushed; public void PushPacket(byte[] bbframe) @@ -30,7 +37,10 @@ public class BbframeDeencapsulator3 : IBbframeDeencapsulator if (mis[bbHeader.Matype2] == null) { logger.InfoFormat("Found a stream on MIS {0}",bbHeader.Matype2); - mis[bbHeader.Matype2] = new GsTypeDetector(); + mis[bbHeader.Matype2] = new GsTypeDetector(bbHeader.Matype2) + { + mpeEventHandler = this._mpeEventHandler + }; } mis[bbHeader.Matype2].PushFrame(bbHeader, new ReadOnlySpan(bbframe, 11, bbframe.Length - 11)); diff --git a/skyscraper8/GS/GSE/GseFragmentation.cs b/skyscraper8/GS/GSE/GseFragmentation.cs new file mode 100644 index 0000000..838a65a --- /dev/null +++ b/skyscraper8/GS/GSE/GseFragmentation.cs @@ -0,0 +1,44 @@ +namespace skyscraper8.GSE.GSE; + +public class GseFragmentation +{ + public GseFragmentation(GsePacket child) + { + this.ProtocolType = child.ProtocolType.Value; + this.Label = child.Label; + AddFragement(child); + } + + private List fragments; + + public void AddFragement(GsePacket packet) + { + if (fragments == null) + fragments = new List(); + + fragments.Add(packet.GseDataBytes); + } + + public GseLabel Label { get; private set; } + + public ushort ProtocolType { get; private set; } + + public bool Validate() + { + //To be implemented... + return true; + } + + public byte[] GetGseDataBytes() + { + int totalLength = fragments.Select(x => x.Length).Sum(); + byte[] buffer = new byte[totalLength]; + int offset = 0; + for (int i = 0; i < fragments.Count; i++) + { + Array.Copy(fragments[i],0,buffer,offset,fragments[i].Length); + offset += fragments[i].Length; + } + return buffer; + } +} \ No newline at end of file diff --git a/skyscraper8/GS/GSE/GseLabel.cs b/skyscraper8/GS/GSE/GseLabel.cs new file mode 100644 index 0000000..563c871 --- /dev/null +++ b/skyscraper8/GS/GSE/GseLabel.cs @@ -0,0 +1,65 @@ +using System.Net.NetworkInformation; + +namespace skyscraper8.GSE.GSE; + +public abstract class GseLabel +{ + public abstract int Length { get; } + public abstract string ToString(); +} + +public class _6byteLabel : GseLabel +{ + public _6byteLabel(byte[] buffer) + { + MAC = new PhysicalAddress(buffer); + } + + public PhysicalAddress MAC { get; private set; } + + override public string ToString() + { + return MAC.ToString(); + } + + public override int Length => 6; +} + +public class _3byteLabel : GseLabel +{ + public _3byteLabel(byte[] buffer) + { + Label = buffer; + } + + public byte[] Label { get; private set; } + + override public string ToString() + { + return BitConverter.ToString(Label); + } + + override public int Length => 3; +} + +public class BroadcastLabel : GseLabel +{ + private BroadcastLabel() {} + + override public string ToString() + { + return ""; + } + + override public int Length => 0; + + private static BroadcastLabel _instance; + public static BroadcastLabel GetInstance() + { + if (_instance == null) + { + _instance = new BroadcastLabel(); + } + return _instance; + } +} \ No newline at end of file diff --git a/skyscraper8/GS/GSE/GsePacket.cs b/skyscraper8/GS/GSE/GsePacket.cs new file mode 100644 index 0000000..0b1bdf5 --- /dev/null +++ b/skyscraper8/GS/GSE/GsePacket.cs @@ -0,0 +1,27 @@ +namespace skyscraper8.GSE.GSE; + +public class GsePacket +{ + public bool StartIndicator { get; } + public bool EndIndicator { get; } + public int LabelTypeIndicator { get; } + public byte? FragmentId { get; set; } + public ushort? TotalLength { get; set; } + public ushort? ProtocolType { get; set; } + public GseLabel Label { get; set; } + public byte[] GseDataBytes { get; set; } + public uint Crc32 { get; set; } + + public GsePacket(bool startIndicator, bool endIndicator, int labelTypeIndicator) + { + StartIndicator = startIndicator; + EndIndicator = endIndicator; + LabelTypeIndicator = labelTypeIndicator; + } + + public override string ToString() + { + return + $"{nameof(StartIndicator)}: {StartIndicator}, {nameof(EndIndicator)}: {EndIndicator}, {nameof(LabelTypeIndicator)}: {LabelTypeIndicator}, {nameof(FragmentId)}: {FragmentId}, {nameof(TotalLength)}: {TotalLength}, {nameof(ProtocolType)}: {ProtocolType}, {nameof(Label)}: {Label}"; + } +} \ No newline at end of file diff --git a/skyscraper8/GS/GSE/GseReader.cs b/skyscraper8/GS/GSE/GseReader.cs new file mode 100644 index 0000000..e2272d1 --- /dev/null +++ b/skyscraper8/GS/GSE/GseReader.cs @@ -0,0 +1,145 @@ +using log4net; +using skyscraper5.Dvb.DataBroadcasting; +using skyscraper5.Skyscraper.IO; + +namespace skyscraper8.GSE.GSE; + +internal class GseReader : IMisHandler +{ + private GseLabel lastLabel; + public IMultiprotocolEncapsulationEventHandler mpeEventHandler { get; set; } + private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name); + public void PushFrame(BBHeader bbHeader, ReadOnlySpan readOnlySpan) + { + MemoryStream ms = new MemoryStream(readOnlySpan.ToArray()); + + while (ms.Position < ms.Length) + { + byte a = ms.ReadUInt8(); + bool startIndicator = (a & 0x80) != 0; + bool endIndicator = (a & 0x40) != 0; + int labelTypeIndicator = (a & 0x30) >> 4; + if (!startIndicator && !endIndicator && labelTypeIndicator == 0) + { + //end of BBFrame + return; + } + else + { + int gseLength = (a & 0x0f); + gseLength <<= 8; + gseLength += ms.ReadUInt8(); + + GsePacket child = new GsePacket(startIndicator, endIndicator, labelTypeIndicator); + if (!startIndicator || !endIndicator) + { + child.FragmentId = ms.ReadUInt8(); + gseLength--; + } + + if (startIndicator && !endIndicator) + { + child.TotalLength = ms.ReadUInt16BE(); + gseLength -= 2; + } + + if (startIndicator) + { + child.ProtocolType = ms.ReadUInt16BE(); + gseLength -= 2; + if (!endIndicator) + child.TotalLength -= 2; + switch (labelTypeIndicator) + { + case 0: + child.Label = new _6byteLabel(ms.ReadBytes(6)); + gseLength -= 6; + if (!endIndicator) + child.TotalLength -= 6; + lastLabel = child.Label; + break; + case 1: + child.Label = new _3byteLabel(ms.ReadBytes(3)); + gseLength -= 3; + if (!endIndicator) + child.TotalLength -= 3; + lastLabel = child.Label; + break; + case 2: + child.Label = BroadcastLabel.GetInstance(); + break; + case 3: + child.Label = this.lastLabel; + break; + default: + throw new NotImplementedException(String.Format("Unknown label type: {0}", labelTypeIndicator)); + } + } + + if (!startIndicator && endIndicator) + gseLength -= 4; //for crc32 + + if (gseLength > ms.GetAvailableBytes()) + return; + child.GseDataBytes = ms.ReadBytes(gseLength); + + if (!startIndicator && endIndicator) + child.Crc32 = ms.ReadUInt32BE(); + + ProcessPacket(child); + } + } + } + + private bool[] warnedEthertypes; + private GseFragmentation[] fragmentations; + private void ProcessPacket(GsePacket child) + { + if (!child.StartIndicator && child.EndIndicator) + { + if (fragmentations == null) + return; + GseFragmentation currentFragmentation = fragmentations[child.FragmentId.Value]; + if (currentFragmentation == null) + return; + currentFragmentation.AddFragement(child); + if (currentFragmentation.Validate()) + { + mpeEventHandler.OnIpDatagram(0x010e,currentFragmentation.GetGseDataBytes()); + } + + fragmentations[child.FragmentId.Value] = null; + return; + } + + if (child.StartIndicator && !child.EndIndicator) + { + if (fragmentations == null) + fragmentations = new GseFragmentation[256]; + fragmentations[child.FragmentId.Value] = new GseFragmentation(child); + return; + } + + if (child.StartIndicator && child.EndIndicator) + { + switch (child.ProtocolType) + { + case 0x0800: + mpeEventHandler.OnIpDatagram(0x010e, child.GseDataBytes); + break; + default: + if (warnedEthertypes == null) + warnedEthertypes = new bool[0xffff]; + if (!warnedEthertypes[child.ProtocolType.Value]) + { + logger.WarnFormat("This GSE contains other traffic types (type {0:X4}) besides IP (type 0x0800). If it's not too much trouble, please consider submitting a sample.",child.ProtocolType.Value); + warnedEthertypes[child.ProtocolType.Value] = true; + } + break; + } + + return; + } + throw new NotImplementedException(child.ToString()); + } +} \ No newline at end of file diff --git a/skyscraper8/GS/GsTypeDetector.cs b/skyscraper8/GS/GsTypeDetector.cs index 3ef824d..8c395e8 100644 --- a/skyscraper8/GS/GsTypeDetector.cs +++ b/skyscraper8/GS/GsTypeDetector.cs @@ -1,9 +1,45 @@ +using log4net; +using skyscraper5.Dvb.DataBroadcasting; +using skyscraper8.GSE.GSE; + namespace skyscraper8.GSE; public class GsTypeDetector : IMisHandler { + public IMultiprotocolEncapsulationEventHandler mpeEventHandler { get; set; } + private readonly byte _misId; + private readonly ILog logger; + private HashSet> _postedStreamTypes; + public GsTypeDetector(byte misId) + { + _misId = misId; + logger = LogManager.GetLogger(String.Format("{0} MIS {1}",System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name,misId)); + } + + private GseReader gseReader; public void PushFrame(BBHeader bbHeader, ReadOnlySpan readOnlySpan) { - throw new NotImplementedException(); + if (bbHeader.TsGs == 1 && bbHeader.SyncByte == 0) + { + //Looks like Continuous GSE. + if (gseReader == null) + { + gseReader = new GseReader(); + gseReader.mpeEventHandler = mpeEventHandler; + } + + gseReader.PushFrame(bbHeader, readOnlySpan); + return; + } + + //We have no idea what this is. + Tuple streamTypeToPost = new Tuple(bbHeader.TsGs, bbHeader.SyncByte); + if (_postedStreamTypes == null) + _postedStreamTypes = new HashSet>(); + if (!_postedStreamTypes.Contains(streamTypeToPost)) + { + logger.WarnFormat("This GS contains packets of type {0} ({2}) with a sync byte of {1}. This is not supported yet. If it isn't too much trouble, please consider sharing a sample of this stream.",streamTypeToPost.Item1,streamTypeToPost.Item2, (TsGsType)streamTypeToPost.Item1); + _postedStreamTypes.Add(streamTypeToPost); + } } } \ No newline at end of file diff --git a/skyscraper8/GS/POC/Pts2Bbf.cs b/skyscraper8/GS/POC/Pts2Bbf.cs index e88fd63..8ebb6d8 100644 --- a/skyscraper8/GS/POC/Pts2Bbf.cs +++ b/skyscraper8/GS/POC/Pts2Bbf.cs @@ -44,7 +44,7 @@ public class Pts2Bbf FileStream fileStream = file.OpenRead(); TsContext mpeg2 = new TsContext(); - mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(dumper)); + mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(null, dumper)); DataStorage dataStorage = new InMemoryScraperStorage(); ObjectStorage objectStorage = new NullObjectStorage(); SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, dataStorage, objectStorage); diff --git a/skyscraper8/GS/POC/Stid135Test.cs b/skyscraper8/GS/POC/Stid135Test.cs index 396c001..5b6b28e 100644 --- a/skyscraper8/GS/POC/Stid135Test.cs +++ b/skyscraper8/GS/POC/Stid135Test.cs @@ -13,13 +13,13 @@ public class Stid135Test { FileStream fileStream = file.OpenRead(); - BbframeDeencapsulator3 decap = new BbframeDeencapsulator3(); - TsContext mpeg2 = new TsContext(); - mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(decap)); DataStorage dataStorage = new InMemoryScraperStorage(); ObjectStorage objectStorage = new NullObjectStorage(); SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, dataStorage, objectStorage); + + mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(skyscraper)); + skyscraper.InitalizeFilterChain(); skyscraper.IngestFromStream(fileStream); diff --git a/skyscraper8/GS/Stid135BbFrameReader.cs b/skyscraper8/GS/Stid135BbFrameReader.cs index 0a1dba5..1859376 100644 --- a/skyscraper8/GS/Stid135BbFrameReader.cs +++ b/skyscraper8/GS/Stid135BbFrameReader.cs @@ -6,6 +6,7 @@ using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; +using skyscraper5.Dvb.DataBroadcasting; namespace skyscraper8.GSE { @@ -13,10 +14,10 @@ namespace skyscraper8.GSE { private IBbframeDeencapsulator deencapsulator; - public Stid135BbFrameReader(IBbframeDeencapsulator deencapsulator = null) + public Stid135BbFrameReader(IMultiprotocolEncapsulationEventHandler mpeEventHandler, IBbframeDeencapsulator deencapsulator = null) { if (deencapsulator == null) - deencapsulator = new BbframeDeencapsulator3(); + deencapsulator = new BbframeDeencapsulator3(mpeEventHandler); this.deencapsulator = deencapsulator; } diff --git a/skyscraper8/GS/TsGsType.cs b/skyscraper8/GS/TsGsType.cs new file mode 100644 index 0000000..94c69cc --- /dev/null +++ b/skyscraper8/GS/TsGsType.cs @@ -0,0 +1,9 @@ +namespace skyscraper8.GSE; + +public enum TsGsType +{ + GenericPacketized, + GenericContinuous, + GseHem, + Transport +} \ No newline at end of file diff --git a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs index 0ecfeb2..e85b738 100644 --- a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs +++ b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs @@ -301,7 +301,7 @@ namespace skyscraper5.Skyscraper.Scraper { if (!DvbContext.IsPidProcessorPresent(0x010e)) { - DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader()); + DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(this)); UiJunction?.SetGseMode(); LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected."); SpecialTsType = 3; @@ -356,7 +356,7 @@ namespace skyscraper5.Skyscraper.Scraper { if (!DvbContext.IsPidProcessorPresent(0x010e)) { - DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader()); + DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(this)); UiJunction?.SetGseMode(); LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected."); SpecialTsType = 3;