Passthrough DVB-SIS DSACI, EIT, PAT and T2-Mi Timestamps to database.

skyscraper8/DvbSis/SisCatContainer.cs

@@ -8,7 +8,7 @@ using System.Threading.Tasks;
 
 namespace skyscraper8.DvbSis
 {
-    internal class SisCatContainer : ICatEventHandler
+    public class SisCatContainer : ICatEventHandler
     {
         public void NotifyOfCaSystem(CaDescriptor caDescriptor, bool fromPmt = false)
         {

skyscraper8/DvbSis/SisEitContainer.cs

@@ -8,7 +8,7 @@ using System.Threading.Tasks;
 
 namespace skyscraper8.DvbSis
 {
-    internal class SisEitContainer : IEitEventHandler
+    public class SisEitContainer : IEitEventHandler
     {
         public void OnEitEvent(EitEvent eitEvent)
         {

skyscraper8/DvbSis/SisHandler.cs

@@ -8,7 +8,7 @@ using System.Threading.Tasks;
 
 namespace skyscraper8.DvbSis
 {
-    internal interface SisHandler
+    public interface SisHandler
     {
         void OnSisCat(int sourcePid, SisCatContainer catContainer);
         void OnSisDsaci(ushort currentDsaGroupId, int versionNumber, byte sectionNumber, byte lastSectionNumber, Stream dsaci);

skyscraper8/DvbSis/SisNitContainer.cs

@@ -8,7 +8,7 @@ using System.Threading.Tasks;
 
 namespace skyscraper8.DvbSis
 {
-    internal class SisNitContainer : INitEventHandler
+    public class SisNitContainer : INitEventHandler
     {
         public void OnNitNetwork(NitNetwork nitNetwork)
         {

skyscraper8/DvbSis/SisPatContainer.cs

@@ -8,7 +8,7 @@ using System.Threading.Tasks;
 
 namespace skyscraper8.DvbSis
 {
-    internal class SisPatContainer : IPatEventHandler
+    public class SisPatContainer : IPatEventHandler
     {
         public ushort? TransportStreamId { get; private set; }
         public int? NetworkPid { get; private set; }

skyscraper8/DvbSis/SisPid.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.DvbSis
+{
+    internal class SisPid
+    {
+        public SisPid(int pid)
+        {
+            this.PID = pid;
+        }
+
+        public int PID { get; }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is SisPid pid &&
+                   PID == pid.PID;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(PID);
+        }
+
+        public override string ToString()
+        {
+            return String.Format("DVB-SIS PID 0x{0:X4}", PID);
+        }
+    }
+}

skyscraper8/DvbSis/SisPmtContainer.cs

@@ -8,7 +8,7 @@ using System.Threading.Tasks;
 
 namespace skyscraper8.DvbSis
 {
-    internal class SisPmtContainer : IPmtEventHandler
+    public class SisPmtContainer : IPmtEventHandler
     {
         public void PmtEvent(ProgramMapping result, int pmtPid)
         {

skyscraper8/DvbSis/SisSdtContainer.cs

@@ -8,7 +8,7 @@ using System.Threading.Tasks;
 
 namespace skyscraper8.DvbSis
 {
-    internal class SisSdtContainer : ISdtEventHandler
+    public class SisSdtContainer : ISdtEventHandler
     {
         public ushort? NetworkId { get; private set; }
         public ushort? TransportStreamId { get; private set; }

skyscraper8/DvbSis/SisTdtContainer.cs

@@ -7,7 +7,7 @@ using System.Threading.Tasks;
 
 namespace skyscraper8.DvbSis
 {
-    internal class SisTdtContainer : ITdtEventHandler
+    public class SisTdtContainer : ITdtEventHandler
     {
         public DateTime? UtcTime { get; private set; }
 

skyscraper8/DvbSis/SisTotContainer.cs

@@ -8,7 +8,7 @@ using System.Threading.Tasks;
 
 namespace skyscraper8.DvbSis
 {
-    internal class SisTotContainer : ITotEventHandler
+    public class SisTotContainer : ITotEventHandler
     {
         public DateTime UtcTime { get; private set; }
         public LocalTimeOffsetDescriptor LocalTimeOffset { get; private set; }

skyscraper8/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "skyscraper8": {
       "commandName": "Project",
-      "commandLineArgs": "\"F:\\utena2\\dvb-sis_badr_12563v.ts\"",
+      "commandLineArgs": "\"F:\\utena2\\dvb-sis_eutelsat5_12522_v.ts\"",
       "remoteDebugEnabled": false
     },
     "Container (Dockerfile)": {

skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs

@@ -98,6 +98,7 @@ using RntParser = skyscraper5.Dvb.TvAnytime.RntParser;
 using skyscraper8.DvbSis;
 using skyscraper8.T2MI.Packets;
 using skyscraper5.Docsis.AnnexC;
+using Ionic.Zlib;
 
 namespace skyscraper5.Skyscraper.Scraper
 {
@@ -107,7 +108,7 @@ namespace skyscraper5.Skyscraper.Scraper
         UpdateNotificationEventHandler, DataCarouselEventHandler, RdsEventHandler, IScte35EventHandler,
         IAutodetectionEventHandler, IRstEventHandler, IRntEventHandler, IMultiprotocolEncapsulationEventHandler, ObjectCarouselEventHandler, T2MIEventHandler,
 		IDisposable, IFrameGrabberEventHandler, IntEventHandler, IRctEventHandler, ISkyscraperContext, IDocsisEventHandler, AbertisDecoderEventHandler, Id3Handler,
-        InteractionChannelHandler, SgtEventHandler, IDvbNipEventHandler, UleEventHandler, OtvSsuHandler, NdsSsuHandler, ISubTsHandler, ILldpFrameHandler
+        InteractionChannelHandler, SgtEventHandler, IDvbNipEventHandler, UleEventHandler, OtvSsuHandler, NdsSsuHandler, ISubTsHandler, ILldpFrameHandler, SisHandler
     {
         public const bool ALLOW_STREAM_TYPE_AUTODETECTION = true;
         public const bool ALLOW_FFMPEG_FRAMEGRABBER = true;
@@ -475,7 +476,7 @@ namespace skyscraper5.Skyscraper.Scraper
             }
 
 			DvbContext.RegisterPacketProcessor(pmtPid, pmtParser);
-            LogEvent(SkyscraperContextEvent.ProgramMapPidFromPat);
+            LogEvent(SkyscraperContextEvent.ProgramMapPidFromPat, String.Format("PMT for program #{0} is on PID 0x{1:X4}", programId, pmtPid));
             UiJunction?.NotifyPatProgram(pmtPid, programId);
 
             if (pmtTracker == null)
@@ -703,14 +704,14 @@ namespace skyscraper5.Skyscraper.Scraper
                             DvbContext.RegisterPacketProcessor(mappingStream.ElementaryPid, new PesDecoder(new Id3PesProcessor(this)));
                             break;
                         case StreamType.SisFramingAndTiming:
-                            DvbContext.RegisterPacketProcessor(mappingStream.ElementaryPid, new FtiHandler(mappingStream.ElementaryPid, new NullSisHandler(), this));
+                            DvbContext.RegisterPacketProcessor(mappingStream.ElementaryPid, new FtiHandler(mappingStream.ElementaryPid, this, this));
                             break;
                         case StreamType.SisDaughterSiteAdapterConfiguration:
-                            DsAciHandler dsaciHandler = new DsAciHandler(new NullSisHandler());
+                            DsAciHandler dsaciHandler = new DsAciHandler(this);
                             DvbContext.RegisterPacketProcessor(mappingStream.ElementaryPid, new PsiDecoder(mappingStream.ElementaryPid, dsaciHandler));
                             break;
                         case StreamType.SisPsiTables:
-                            SisPsiHandler sisPsiHandler = new SisPsiHandler(new NullSisHandler());
+                            SisPsiHandler sisPsiHandler = new SisPsiHandler(this);
                             DvbContext.RegisterPacketProcessor(mappingStream.ElementaryPid, new PsiDecoder(mappingStream.ElementaryPid, sisPsiHandler));
                             break;
                         default:
@@ -3325,5 +3326,153 @@ namespace skyscraper5.Skyscraper.Scraper
         {
             logger.WarnFormat("Found a DOCSIS Dynamic Service Addition. Those aren't supported yet. It would be great if you could share a sample of this stream, so those can be implemented.");
         }
+
+        private SkyscraperContext GetSisContext(int sourcePid)
+        {
+            if (subSkyscrapers == null)
+                subSkyscrapers = new Dictionary<object, SkyscraperContext>();
+
+            SisPid sisContext = new SisPid(sourcePid);
+            if (subSkyscrapers.ContainsKey(sisContext))
+            {
+                return subSkyscrapers[sisContext];
+            }
+            else
+            {
+                LogEvent(SkyscraperContextEvent.DvbSisDaughterSitePsi, String.Format("Additional terrestrial PSI/SI tables on PID 0x{0:X4}", sourcePid));
+                SkyscraperContext child = new SkyscraperContext(new TsContext(), this.DataStorage, this.ObjectStorage);
+                child.IsChild = true;
+                child.ChildName = sisContext.ToString();
+                subSkyscrapers.Add(sisContext, child);
+                return child;
+            }
+        }
+
+        public void OnSisCat(int sourcePid, SisCatContainer catContainer)
+        {
+            throw new NotImplementedException();
+        }
+
+        public void OnSisDsaci(ushort currentDsaGroupId, int versionNumber, byte sectionNumber, byte lastSectionNumber, Stream dsaci)
+        {
+            if (!CurrentNetworkId.HasValue)
+                return;
+
+            if (!CurrentTransportStreamId.HasValue)
+                return;
+
+            if (!ObjectStorage.TestForSisDsaci(CurrentNetworkId.Value,CurrentTransportStreamId.Value, currentDsaGroupId, versionNumber))
+            {
+                dsaci.Position = 0;
+                byte gzMagicA = dsaci.ReadUInt8();
+                byte gzMagicB = dsaci.ReadUInt8();
+                dsaci.Position -= 2;
+                if (gzMagicA == 0x1f &&  gzMagicB == 0x8b)
+                {
+                    dsaci = new GZipStream(dsaci, CompressionMode.Decompress);
+                }
+
+                LogEvent(SkyscraperContextEvent.DvbSisDaughterSiteAdapterConfiguration, String.Format("Group ID = {0}, Version = {1}", currentDsaGroupId, versionNumber));
+                ObjectStorage.StoreSisDsaci(CurrentNetworkId.Value, CurrentTransportStreamId.Value, currentDsaGroupId, versionNumber, dsaci);
+                dsaci.Dispose();
+            }
+        }
+
+        public void OnSisEit(int sourcePid, SisEitContainer eitContainer)
+        {
+            SkyscraperContext skyscraperContext = GetSisContext(sourcePid);
+            if (eitContainer.NetworkId.HasValue)
+                skyscraperContext.SetNetworkId(eitContainer.NetworkId.Value);
+            if (eitContainer.TransportStreamId.HasValue)
+                skyscraperContext.SetTransportStreamId(eitContainer.TransportStreamId.Value);
+
+            foreach (EitEvent eitEvent in eitContainer.Events)
+            {
+                skyscraperContext.OnEitEvent(eitEvent);
+            }
+        }
+
+        private bool[] _sisFtiFlags;
+        private _0xF0_FramingTimingInformation.Plp[][] _sisFtiData;
+        public void OnSisFti(int relatedPid, _0xF0_FramingTimingInformation fti)
+        {
+            if (_sisFtiFlags == null)
+            {
+                _sisFtiFlags = new bool[0x1fff];
+                _sisFtiData = new _0xF0_FramingTimingInformation.Plp[0x1fff][];
+            }
+
+            if (!_sisFtiFlags[relatedPid])
+            {
+                LogEvent(SkyscraperContextEvent.DvbSisFramingInformation, String.Format("on PID 0x{0:X4}", relatedPid));
+                _sisFtiFlags[relatedPid] = true;
+            }
+            _sisFtiData[relatedPid] = fti.Plps;
+        }
+
+        public void OnSisNit(int sourcePid, SisNitContainer nitContainer)
+        {
+            throw new NotImplementedException();
+        }
+
+        public void OnSisPat(int sourcePid, SisPatContainer patContainer)
+        {
+            SkyscraperContext skyscraperContext = GetSisContext(sourcePid);
+            if (patContainer.NetworkPid.HasValue)
+                skyscraperContext.NetworkPidFromPat(patContainer.NetworkPid.Value);
+            if (patContainer.TransportStreamId.HasValue)
+                skyscraperContext.SetTransportStreamId(patContainer.TransportStreamId.Value);
+
+            foreach ( KeyValuePair<int, ushort> program in patContainer.ProgramMappings)
+            {
+                skyscraperContext.ProgramMapPidFromPat(program.Key, program.Value);
+            }
+
+        }
+
+        public void OnSisPmt(int sourcePid, SisPmtContainer pmtContainer)
+        {
+            throw new NotImplementedException();
+        }
+
+        public void OnSisSdt(int sourcePid, SisSdtContainer sdtContainer)
+        {
+            throw new NotImplementedException();
+        }
+
+        public void OnSisTdt(int sourcePid, SisTdtContainer tdtContainer)
+        {
+            throw new NotImplementedException();
+        }
+
+        private bool[] _sisTimestampFlags;
+        public void OnSisTimestamp(int pid, _0x20_DvbT2Timestamp t2Timestamp)
+        {
+            if (!CurrentNetworkId.HasValue)
+                return;
+
+            if (!CurrentTransportStreamId.HasValue)
+                return;
+
+            DateTime resolveTime = t2Timestamp.ResolveTime();
+            DateTime knownTimestamp = DataStorage.T2MiGetTimestamp(CurrentNetworkId.Value, CurrentTransportStreamId.Value, pid);
+            if (resolveTime > knownTimestamp)
+            {
+                if (_sisTimestampFlags == null)
+                    _sisTimestampFlags = new bool[0x1fff];
+
+                if (!_sisTimestampFlags[pid])
+                {
+                    LogEvent(SkyscraperContextEvent.DvbSisTimestamp, t2Timestamp.ResolveTime().ToString());
+                    _sisTimestampFlags[pid] = true;
+                }
+                DataStorage.T2MiSetTimestamp(CurrentNetworkId.Value, CurrentTransportStreamId.Value, pid, resolveTime);
+            }
+        }
+
+        public void OnSisTot(int sourcePid, SisTotContainer totContainer)
+        {
+            throw new NotImplementedException();
+        }
     }
 }

skyscraper8/Skyscraper/Scraper/SkyscraperContextEvent.cs

@@ -100,6 +100,10 @@
 		TimLowerLayerService,
 		TimHigherLayerInitalization,
 		TimLogonResponse,
-		TimForwardInteractionPath
+		TimForwardInteractionPath,
+        DvbSisDaughterSitePsi,
+        DvbSisFramingInformation,
+        DvbSisTimestamp,
+        DvbSisDaughterSiteAdapterConfiguration
     }
 }

skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Net;
@@ -1977,6 +1978,26 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
 		public byte[] SsdpGetMetadata(SsdpDevice ssdpDevice)
 		{
 			throw new NotImplementedException();
-		}
+		}
+
+        public bool TestForSisDsaci(int networkId, int tsId, ushort groupId, int versionNumber)
+        {
+            string xmlFileName = Path.Combine(rootDirectory.FullName, "DVB-SIS", networkId.ToString(), tsId.ToString(), String.Format("DSACI_Group{0}_Version{1}.xml"));
+            FileInfo fi = new FileInfo(xmlFileName);
+            return fi.Exists;
+        }
+
+        public void StoreSisDsaci(int networkId, int tsId, ushort currentDsaGroupId, int versionNumber, Stream dsaci)
+        {
+            string xmlFileName = Path.Combine(rootDirectory.FullName, "DVB-SIS", networkId.ToString(), tsId.ToString(), String.Format("DSACI_Group{0}_Version{1}.xml"));
+            FileInfo fi = new FileInfo(xmlFileName);
+            fi.Directory.EnsureExists();
+
+            fi.Refresh();
+            FileStream fileStream = fi.OpenWrite();
+            dsaci.CopyTo(fileStream);
+            fileStream.Flush();
+            fileStream.Close();
+        }
     }
 }

skyscraper8/Skyscraper/Scraper/Storage/ObjectStorage.cs

@@ -36,6 +36,8 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
         bool OtvSsuTestFile(int? currentNetworkId, int? currentTransportStreamId, int sourcePid, ushort tableIdExtension, uint fileId, uint unknown1, uint length);
         void OnOtvSsuComplete(int? currentNetworkId, int? currentTransportStreamId, int sourcePid, Stream getStream, ushort tableIdExtension, uint fileId, uint unknown1, uint length);
         void OnNdsSsuComplete(int? currentNetworkId, int? currentTransportStreamId, int pid, ushort tableIdExtension, NdsSsuDataMap dataMap);
-        bool NdsSsuTestFile(int? currentNetworkId, int? currentTransportStreamId, int pid, ushort tableIdExtension);
+        bool NdsSsuTestFile(int? currentNetworkId, int? currentTransportStreamId, int pid, ushort tableIdExtension);
+        bool TestForSisDsaci(int value1, int value2, ushort groupId, int versionNumber);
+        void StoreSisDsaci(int value1, int value2, ushort currentDsaGroupId, int versionNumber, Stream dsaci);
     }
 }

skyscraper8/T2MI/Packets/0xF0_FramingTimingInformation.cs

@@ -12,7 +12,7 @@ namespace skyscraper8.T2MI.Packets
 {
     [SkyscraperPlugin]
     [T2MiPacketType(0xf0)]
-    internal class _0xF0_FramingTimingInformation : T2MiPacket
+    public class _0xF0_FramingTimingInformation : T2MiPacket
     {
         public _0xF0_FramingTimingInformation(T2MIHeader header, byte[] buffer) : base(header, buffer)
         {