From 7ff359e52ddcf9922cfbe095e1c03d4911c9b6e7 Mon Sep 17 00:00:00 2001 From: Fey Date: Wed, 17 Dec 2025 09:28:53 +0100 Subject: [PATCH] Passthrough DVB-SIS DSACI, EIT, PAT and T2-Mi Timestamps to database. --- skyscraper8/DvbSis/SisCatContainer.cs | 2 +- skyscraper8/DvbSis/SisEitContainer.cs | 2 +- skyscraper8/DvbSis/SisHandler.cs | 2 +- skyscraper8/DvbSis/SisNitContainer.cs | 2 +- skyscraper8/DvbSis/SisPatContainer.cs | 2 +- skyscraper8/DvbSis/SisPid.cs | 34 ++++ skyscraper8/DvbSis/SisPmtContainer.cs | 2 +- skyscraper8/DvbSis/SisSdtContainer.cs | 2 +- skyscraper8/DvbSis/SisTdtContainer.cs | 2 +- skyscraper8/DvbSis/SisTotContainer.cs | 2 +- skyscraper8/Properties/launchSettings.json | 2 +- .../Skyscraper/Scraper/SkyscraperContext.cs | 159 +++++++++++++++++- .../Scraper/SkyscraperContextEvent.cs | 6 +- .../Storage/Filesystem/FilesystemStorage.cs | 23 ++- .../Scraper/Storage/ObjectStorage.cs | 4 +- .../Packets/0xF0_FramingTimingInformation.cs | 2 +- 16 files changed, 229 insertions(+), 19 deletions(-) create mode 100644 skyscraper8/DvbSis/SisPid.cs diff --git a/skyscraper8/DvbSis/SisCatContainer.cs b/skyscraper8/DvbSis/SisCatContainer.cs index d0976c3..294674b 100644 --- a/skyscraper8/DvbSis/SisCatContainer.cs +++ b/skyscraper8/DvbSis/SisCatContainer.cs @@ -8,7 +8,7 @@ using System.Threading.Tasks; namespace skyscraper8.DvbSis { - internal class SisCatContainer : ICatEventHandler + public class SisCatContainer : ICatEventHandler { public void NotifyOfCaSystem(CaDescriptor caDescriptor, bool fromPmt = false) { diff --git a/skyscraper8/DvbSis/SisEitContainer.cs b/skyscraper8/DvbSis/SisEitContainer.cs index 56949fc..a98fc64 100644 --- a/skyscraper8/DvbSis/SisEitContainer.cs +++ b/skyscraper8/DvbSis/SisEitContainer.cs @@ -8,7 +8,7 @@ using System.Threading.Tasks; namespace skyscraper8.DvbSis { - internal class SisEitContainer : IEitEventHandler + public class SisEitContainer : IEitEventHandler { public void OnEitEvent(EitEvent eitEvent) { diff --git a/skyscraper8/DvbSis/SisHandler.cs b/skyscraper8/DvbSis/SisHandler.cs index f33320e..1225cfb 100644 --- a/skyscraper8/DvbSis/SisHandler.cs +++ b/skyscraper8/DvbSis/SisHandler.cs @@ -8,7 +8,7 @@ using System.Threading.Tasks; namespace skyscraper8.DvbSis { - internal interface SisHandler + public interface SisHandler { void OnSisCat(int sourcePid, SisCatContainer catContainer); void OnSisDsaci(ushort currentDsaGroupId, int versionNumber, byte sectionNumber, byte lastSectionNumber, Stream dsaci); diff --git a/skyscraper8/DvbSis/SisNitContainer.cs b/skyscraper8/DvbSis/SisNitContainer.cs index 707b59e..14164ae 100644 --- a/skyscraper8/DvbSis/SisNitContainer.cs +++ b/skyscraper8/DvbSis/SisNitContainer.cs @@ -8,7 +8,7 @@ using System.Threading.Tasks; namespace skyscraper8.DvbSis { - internal class SisNitContainer : INitEventHandler + public class SisNitContainer : INitEventHandler { public void OnNitNetwork(NitNetwork nitNetwork) { diff --git a/skyscraper8/DvbSis/SisPatContainer.cs b/skyscraper8/DvbSis/SisPatContainer.cs index 4c2a72a..65db021 100644 --- a/skyscraper8/DvbSis/SisPatContainer.cs +++ b/skyscraper8/DvbSis/SisPatContainer.cs @@ -8,7 +8,7 @@ using System.Threading.Tasks; namespace skyscraper8.DvbSis { - internal class SisPatContainer : IPatEventHandler + public class SisPatContainer : IPatEventHandler { public ushort? TransportStreamId { get; private set; } public int? NetworkPid { get; private set; } diff --git a/skyscraper8/DvbSis/SisPid.cs b/skyscraper8/DvbSis/SisPid.cs new file mode 100644 index 0000000..5f80b6b --- /dev/null +++ b/skyscraper8/DvbSis/SisPid.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.DvbSis +{ + internal class SisPid + { + public SisPid(int pid) + { + this.PID = pid; + } + + public int PID { get; } + + public override bool Equals(object? obj) + { + return obj is SisPid pid && + PID == pid.PID; + } + + public override int GetHashCode() + { + return HashCode.Combine(PID); + } + + public override string ToString() + { + return String.Format("DVB-SIS PID 0x{0:X4}", PID); + } + } +} diff --git a/skyscraper8/DvbSis/SisPmtContainer.cs b/skyscraper8/DvbSis/SisPmtContainer.cs index 33b0f6b..2094f59 100644 --- a/skyscraper8/DvbSis/SisPmtContainer.cs +++ b/skyscraper8/DvbSis/SisPmtContainer.cs @@ -8,7 +8,7 @@ using System.Threading.Tasks; namespace skyscraper8.DvbSis { - internal class SisPmtContainer : IPmtEventHandler + public class SisPmtContainer : IPmtEventHandler { public void PmtEvent(ProgramMapping result, int pmtPid) { diff --git a/skyscraper8/DvbSis/SisSdtContainer.cs b/skyscraper8/DvbSis/SisSdtContainer.cs index 59fecee..f4006de 100644 --- a/skyscraper8/DvbSis/SisSdtContainer.cs +++ b/skyscraper8/DvbSis/SisSdtContainer.cs @@ -8,7 +8,7 @@ using System.Threading.Tasks; namespace skyscraper8.DvbSis { - internal class SisSdtContainer : ISdtEventHandler + public class SisSdtContainer : ISdtEventHandler { public ushort? NetworkId { get; private set; } public ushort? TransportStreamId { get; private set; } diff --git a/skyscraper8/DvbSis/SisTdtContainer.cs b/skyscraper8/DvbSis/SisTdtContainer.cs index db0579a..be0c77c 100644 --- a/skyscraper8/DvbSis/SisTdtContainer.cs +++ b/skyscraper8/DvbSis/SisTdtContainer.cs @@ -7,7 +7,7 @@ using System.Threading.Tasks; namespace skyscraper8.DvbSis { - internal class SisTdtContainer : ITdtEventHandler + public class SisTdtContainer : ITdtEventHandler { public DateTime? UtcTime { get; private set; } diff --git a/skyscraper8/DvbSis/SisTotContainer.cs b/skyscraper8/DvbSis/SisTotContainer.cs index b6eea9a..ad7b896 100644 --- a/skyscraper8/DvbSis/SisTotContainer.cs +++ b/skyscraper8/DvbSis/SisTotContainer.cs @@ -8,7 +8,7 @@ using System.Threading.Tasks; namespace skyscraper8.DvbSis { - internal class SisTotContainer : ITotEventHandler + public class SisTotContainer : ITotEventHandler { public DateTime UtcTime { get; private set; } public LocalTimeOffsetDescriptor LocalTimeOffset { get; private set; } diff --git a/skyscraper8/Properties/launchSettings.json b/skyscraper8/Properties/launchSettings.json index c3ff4ae..f8cec80 100644 --- a/skyscraper8/Properties/launchSettings.json +++ b/skyscraper8/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "skyscraper8": { "commandName": "Project", - "commandLineArgs": "\"F:\\utena2\\dvb-sis_badr_12563v.ts\"", + "commandLineArgs": "\"F:\\utena2\\dvb-sis_eutelsat5_12522_v.ts\"", "remoteDebugEnabled": false }, "Container (Dockerfile)": { diff --git a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs index 27df39f..cd0b408 100644 --- a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs +++ b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs @@ -98,6 +98,7 @@ using RntParser = skyscraper5.Dvb.TvAnytime.RntParser; using skyscraper8.DvbSis; using skyscraper8.T2MI.Packets; using skyscraper5.Docsis.AnnexC; +using Ionic.Zlib; namespace skyscraper5.Skyscraper.Scraper { @@ -107,7 +108,7 @@ namespace skyscraper5.Skyscraper.Scraper UpdateNotificationEventHandler, DataCarouselEventHandler, RdsEventHandler, IScte35EventHandler, IAutodetectionEventHandler, IRstEventHandler, IRntEventHandler, IMultiprotocolEncapsulationEventHandler, ObjectCarouselEventHandler, T2MIEventHandler, IDisposable, IFrameGrabberEventHandler, IntEventHandler, IRctEventHandler, ISkyscraperContext, IDocsisEventHandler, AbertisDecoderEventHandler, Id3Handler, - InteractionChannelHandler, SgtEventHandler, IDvbNipEventHandler, UleEventHandler, OtvSsuHandler, NdsSsuHandler, ISubTsHandler, ILldpFrameHandler + InteractionChannelHandler, SgtEventHandler, IDvbNipEventHandler, UleEventHandler, OtvSsuHandler, NdsSsuHandler, ISubTsHandler, ILldpFrameHandler, SisHandler { public const bool ALLOW_STREAM_TYPE_AUTODETECTION = true; public const bool ALLOW_FFMPEG_FRAMEGRABBER = true; @@ -475,7 +476,7 @@ namespace skyscraper5.Skyscraper.Scraper } DvbContext.RegisterPacketProcessor(pmtPid, pmtParser); - LogEvent(SkyscraperContextEvent.ProgramMapPidFromPat); + LogEvent(SkyscraperContextEvent.ProgramMapPidFromPat, String.Format("PMT for program #{0} is on PID 0x{1:X4}", programId, pmtPid)); UiJunction?.NotifyPatProgram(pmtPid, programId); if (pmtTracker == null) @@ -703,14 +704,14 @@ namespace skyscraper5.Skyscraper.Scraper DvbContext.RegisterPacketProcessor(mappingStream.ElementaryPid, new PesDecoder(new Id3PesProcessor(this))); break; case StreamType.SisFramingAndTiming: - DvbContext.RegisterPacketProcessor(mappingStream.ElementaryPid, new FtiHandler(mappingStream.ElementaryPid, new NullSisHandler(), this)); + DvbContext.RegisterPacketProcessor(mappingStream.ElementaryPid, new FtiHandler(mappingStream.ElementaryPid, this, this)); break; case StreamType.SisDaughterSiteAdapterConfiguration: - DsAciHandler dsaciHandler = new DsAciHandler(new NullSisHandler()); + DsAciHandler dsaciHandler = new DsAciHandler(this); DvbContext.RegisterPacketProcessor(mappingStream.ElementaryPid, new PsiDecoder(mappingStream.ElementaryPid, dsaciHandler)); break; case StreamType.SisPsiTables: - SisPsiHandler sisPsiHandler = new SisPsiHandler(new NullSisHandler()); + SisPsiHandler sisPsiHandler = new SisPsiHandler(this); DvbContext.RegisterPacketProcessor(mappingStream.ElementaryPid, new PsiDecoder(mappingStream.ElementaryPid, sisPsiHandler)); break; default: @@ -3325,5 +3326,153 @@ namespace skyscraper5.Skyscraper.Scraper { logger.WarnFormat("Found a DOCSIS Dynamic Service Addition. Those aren't supported yet. It would be great if you could share a sample of this stream, so those can be implemented."); } + + private SkyscraperContext GetSisContext(int sourcePid) + { + if (subSkyscrapers == null) + subSkyscrapers = new Dictionary(); + + SisPid sisContext = new SisPid(sourcePid); + if (subSkyscrapers.ContainsKey(sisContext)) + { + return subSkyscrapers[sisContext]; + } + else + { + LogEvent(SkyscraperContextEvent.DvbSisDaughterSitePsi, String.Format("Additional terrestrial PSI/SI tables on PID 0x{0:X4}", sourcePid)); + SkyscraperContext child = new SkyscraperContext(new TsContext(), this.DataStorage, this.ObjectStorage); + child.IsChild = true; + child.ChildName = sisContext.ToString(); + subSkyscrapers.Add(sisContext, child); + return child; + } + } + + public void OnSisCat(int sourcePid, SisCatContainer catContainer) + { + throw new NotImplementedException(); + } + + public void OnSisDsaci(ushort currentDsaGroupId, int versionNumber, byte sectionNumber, byte lastSectionNumber, Stream dsaci) + { + if (!CurrentNetworkId.HasValue) + return; + + if (!CurrentTransportStreamId.HasValue) + return; + + if (!ObjectStorage.TestForSisDsaci(CurrentNetworkId.Value,CurrentTransportStreamId.Value, currentDsaGroupId, versionNumber)) + { + dsaci.Position = 0; + byte gzMagicA = dsaci.ReadUInt8(); + byte gzMagicB = dsaci.ReadUInt8(); + dsaci.Position -= 2; + if (gzMagicA == 0x1f && gzMagicB == 0x8b) + { + dsaci = new GZipStream(dsaci, CompressionMode.Decompress); + } + + LogEvent(SkyscraperContextEvent.DvbSisDaughterSiteAdapterConfiguration, String.Format("Group ID = {0}, Version = {1}", currentDsaGroupId, versionNumber)); + ObjectStorage.StoreSisDsaci(CurrentNetworkId.Value, CurrentTransportStreamId.Value, currentDsaGroupId, versionNumber, dsaci); + dsaci.Dispose(); + } + } + + public void OnSisEit(int sourcePid, SisEitContainer eitContainer) + { + SkyscraperContext skyscraperContext = GetSisContext(sourcePid); + if (eitContainer.NetworkId.HasValue) + skyscraperContext.SetNetworkId(eitContainer.NetworkId.Value); + if (eitContainer.TransportStreamId.HasValue) + skyscraperContext.SetTransportStreamId(eitContainer.TransportStreamId.Value); + + foreach (EitEvent eitEvent in eitContainer.Events) + { + skyscraperContext.OnEitEvent(eitEvent); + } + } + + private bool[] _sisFtiFlags; + private _0xF0_FramingTimingInformation.Plp[][] _sisFtiData; + public void OnSisFti(int relatedPid, _0xF0_FramingTimingInformation fti) + { + if (_sisFtiFlags == null) + { + _sisFtiFlags = new bool[0x1fff]; + _sisFtiData = new _0xF0_FramingTimingInformation.Plp[0x1fff][]; + } + + if (!_sisFtiFlags[relatedPid]) + { + LogEvent(SkyscraperContextEvent.DvbSisFramingInformation, String.Format("on PID 0x{0:X4}", relatedPid)); + _sisFtiFlags[relatedPid] = true; + } + _sisFtiData[relatedPid] = fti.Plps; + } + + public void OnSisNit(int sourcePid, SisNitContainer nitContainer) + { + throw new NotImplementedException(); + } + + public void OnSisPat(int sourcePid, SisPatContainer patContainer) + { + SkyscraperContext skyscraperContext = GetSisContext(sourcePid); + if (patContainer.NetworkPid.HasValue) + skyscraperContext.NetworkPidFromPat(patContainer.NetworkPid.Value); + if (patContainer.TransportStreamId.HasValue) + skyscraperContext.SetTransportStreamId(patContainer.TransportStreamId.Value); + + foreach ( KeyValuePair program in patContainer.ProgramMappings) + { + skyscraperContext.ProgramMapPidFromPat(program.Key, program.Value); + } + + } + + public void OnSisPmt(int sourcePid, SisPmtContainer pmtContainer) + { + throw new NotImplementedException(); + } + + public void OnSisSdt(int sourcePid, SisSdtContainer sdtContainer) + { + throw new NotImplementedException(); + } + + public void OnSisTdt(int sourcePid, SisTdtContainer tdtContainer) + { + throw new NotImplementedException(); + } + + private bool[] _sisTimestampFlags; + public void OnSisTimestamp(int pid, _0x20_DvbT2Timestamp t2Timestamp) + { + if (!CurrentNetworkId.HasValue) + return; + + if (!CurrentTransportStreamId.HasValue) + return; + + DateTime resolveTime = t2Timestamp.ResolveTime(); + DateTime knownTimestamp = DataStorage.T2MiGetTimestamp(CurrentNetworkId.Value, CurrentTransportStreamId.Value, pid); + if (resolveTime > knownTimestamp) + { + if (_sisTimestampFlags == null) + _sisTimestampFlags = new bool[0x1fff]; + + if (!_sisTimestampFlags[pid]) + { + LogEvent(SkyscraperContextEvent.DvbSisTimestamp, t2Timestamp.ResolveTime().ToString()); + _sisTimestampFlags[pid] = true; + } + DataStorage.T2MiSetTimestamp(CurrentNetworkId.Value, CurrentTransportStreamId.Value, pid, resolveTime); + } + } + + public void OnSisTot(int sourcePid, SisTotContainer totContainer) + { + throw new NotImplementedException(); + } } } diff --git a/skyscraper8/Skyscraper/Scraper/SkyscraperContextEvent.cs b/skyscraper8/Skyscraper/Scraper/SkyscraperContextEvent.cs index bec8dc7..a734ba2 100644 --- a/skyscraper8/Skyscraper/Scraper/SkyscraperContextEvent.cs +++ b/skyscraper8/Skyscraper/Scraper/SkyscraperContextEvent.cs @@ -100,6 +100,10 @@ TimLowerLayerService, TimHigherLayerInitalization, TimLogonResponse, - TimForwardInteractionPath + TimForwardInteractionPath, + DvbSisDaughterSitePsi, + DvbSisFramingInformation, + DvbSisTimestamp, + DvbSisDaughterSiteAdapterConfiguration } } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs index 40b327c..d9dc24d 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Net; @@ -1977,6 +1978,26 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem public byte[] SsdpGetMetadata(SsdpDevice ssdpDevice) { throw new NotImplementedException(); - } + } + + public bool TestForSisDsaci(int networkId, int tsId, ushort groupId, int versionNumber) + { + string xmlFileName = Path.Combine(rootDirectory.FullName, "DVB-SIS", networkId.ToString(), tsId.ToString(), String.Format("DSACI_Group{0}_Version{1}.xml")); + FileInfo fi = new FileInfo(xmlFileName); + return fi.Exists; + } + + public void StoreSisDsaci(int networkId, int tsId, ushort currentDsaGroupId, int versionNumber, Stream dsaci) + { + string xmlFileName = Path.Combine(rootDirectory.FullName, "DVB-SIS", networkId.ToString(), tsId.ToString(), String.Format("DSACI_Group{0}_Version{1}.xml")); + FileInfo fi = new FileInfo(xmlFileName); + fi.Directory.EnsureExists(); + + fi.Refresh(); + FileStream fileStream = fi.OpenWrite(); + dsaci.CopyTo(fileStream); + fileStream.Flush(); + fileStream.Close(); + } } } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/ObjectStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/ObjectStorage.cs index 41511a8..a813921 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/ObjectStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/ObjectStorage.cs @@ -36,6 +36,8 @@ namespace skyscraper8.Skyscraper.Scraper.Storage bool OtvSsuTestFile(int? currentNetworkId, int? currentTransportStreamId, int sourcePid, ushort tableIdExtension, uint fileId, uint unknown1, uint length); void OnOtvSsuComplete(int? currentNetworkId, int? currentTransportStreamId, int sourcePid, Stream getStream, ushort tableIdExtension, uint fileId, uint unknown1, uint length); void OnNdsSsuComplete(int? currentNetworkId, int? currentTransportStreamId, int pid, ushort tableIdExtension, NdsSsuDataMap dataMap); - bool NdsSsuTestFile(int? currentNetworkId, int? currentTransportStreamId, int pid, ushort tableIdExtension); + bool NdsSsuTestFile(int? currentNetworkId, int? currentTransportStreamId, int pid, ushort tableIdExtension); + bool TestForSisDsaci(int value1, int value2, ushort groupId, int versionNumber); + void StoreSisDsaci(int value1, int value2, ushort currentDsaGroupId, int versionNumber, Stream dsaci); } } diff --git a/skyscraper8/T2MI/Packets/0xF0_FramingTimingInformation.cs b/skyscraper8/T2MI/Packets/0xF0_FramingTimingInformation.cs index ad36950..4bd301a 100644 --- a/skyscraper8/T2MI/Packets/0xF0_FramingTimingInformation.cs +++ b/skyscraper8/T2MI/Packets/0xF0_FramingTimingInformation.cs @@ -12,7 +12,7 @@ namespace skyscraper8.T2MI.Packets { [SkyscraperPlugin] [T2MiPacketType(0xf0)] - internal class _0xF0_FramingTimingInformation : T2MiPacket + public class _0xF0_FramingTimingInformation : T2MiPacket { public _0xF0_FramingTimingInformation(T2MIHeader header, byte[] buffer) : base(header, buffer) {