Separated DVB PMT and ISDB PMT.

skyscraper8/Dvb/Descriptors/0x52_StreamIdentifierDescriptor.cs

@@ -9,7 +9,7 @@ using skyscraper5.Skyscraper.Plugins;
 namespace skyscraper5.Dvb.Descriptors
 {
 	[SkyscraperPlugin]
-	[TsDescriptor(0x52,"PMT")]
+	[TsDescriptor(0x52,"PMT", "ISDB PMT")]
     [BannedTable("TSDT","EIT")]
     class StreamIdentifierDescriptor : TsDescriptor
     {

skyscraper8/Isdb/Descriptors/0xFD_DataComponentDescriptor.cs

@@ -0,0 +1,21 @@
+using skyscraper5.Mpeg2;
+using skyscraper5.Skyscraper.IO;
+using skyscraper5.Skyscraper.Plugins;
+
+namespace skyscraper8.Isdb.Descriptors;
+
+[SkyscraperPlugin]
+[TsDescriptor(0xfd,"ISDB PMT")]
+public class _0xFD_DataComponentDescriptor : TsDescriptor {
+    public _0xFD_DataComponentDescriptor(byte[] buffer)
+    {
+        MemoryStream ms = new MemoryStream(buffer);
+        DataComponentId = ms.ReadUInt16BE();
+        AdditionalDataComponentInfo = ms.ReadBytes(ms.GetAvailableBytes());
+        Valid = true;
+    }
+
+    public byte[] AdditionalDataComponentInfo { get; set; }
+
+    public ushort DataComponentId { get; private set; }
+}

skyscraper8/Isdb/EventHandlers/IsdbPmtEventHandler.cs

@@ -0,0 +1,20 @@
+using skyscraper5.Mpeg2.Psi.Model;
+using skyscraper8.Isdb.Tables;
+
+namespace skyscraper8.Isdb.EventHandlers;
+
+public interface IIsdbPmtEventHandler
+{
+    void PmtEvent(IsdbPmt result, int sourcePid);
+    void PmtError(IsdbPmtErrors esInfoLengthLongerThanBuffer, int pmtPid, ushort programNumber);
+}
+
+public enum IsdbPmtErrors
+{
+    EsInfoLengthLongerThanBuffer,
+    DescriptorBufferLongerThanSourceBuffer,
+    ProgramNumberMismatch,
+    SectionSyntaxIndicatorMissing,
+    WrongTableId,
+    SourcePidPmtPidMismatch
+}

skyscraper8/Isdb/Matenro.cs

@@ -9,12 +9,14 @@ using skyscraper5.Mpeg2.Psi;
 using skyscraper5.Mpeg2.Psi.Model;
 using skyscraper5.Skyscraper.IO.CrazycatStreamReader;
 using skyscraper5.Skyscraper.Scraper;
+using skyscraper8.Isdb.EventHandlers;
 using skyscraper8.Isdb.Psi;
+using skyscraper8.Isdb.Tables;
 using skyscraper8.Skyscraper.Scraper.Storage;
 
 namespace skyscraper8.Isdb;
 
-public class Matenro : IPatEventHandler, ICatEventHandler, IPmtEventHandler, IBatEventHandler, ISdtEventHandler, IEitEventHandler, IRstEventHandler, ITdtEventHandler, ITotEventHandler
+public class Matenro : IPatEventHandler, ICatEventHandler, IBatEventHandler, ISdtEventHandler, IEitEventHandler, IRstEventHandler, ITdtEventHandler, ITotEventHandler, IIsdbPmtEventHandler
 {
     public TsContext DvbContext { get; }
     public DataStorage DataStorage { get; }
@@ -36,7 +38,7 @@ public class Matenro : IPatEventHandler, ICatEventHandler, IPmtEventHandler, IBa
         UiJunction = uiJunction;
         LogEvent = logEvent;
 
-        logger = LogManager.GetLogger(this.GetType());
+        logger = LogManager.GetLogger(this.GetType().Name);
 
         
         Pid0x11Decoder pid0X11Decoder = new Pid0x11Decoder(this, this);
@@ -98,8 +100,7 @@ public class Matenro : IPatEventHandler, ICatEventHandler, IPmtEventHandler, IBa
 
         if (pmtPidsAssociated[pmtPid] == false)
         {
-            PmtParser pmtParser = new PmtParser(this);
-            pmtParser.SetStandard(STD_TYPE.STD_ISDBT);
+            IsdbPmtParser pmtParser = new IsdbPmtParser(this, pmtPid, programNumber);
             DvbContext.RegisterPacketProcessor(pmtPid, new PsiDecoder(pmtPid, pmtParser));
             logger.InfoFormat("PAT points to a PMT on PID 0x{0:X4}.", pmtPid);
             pmtPidsAssociated[pmtPid] = true;
@@ -140,11 +141,6 @@ public class Matenro : IPatEventHandler, ICatEventHandler, IPmtEventHandler, IBa
         throw new NotImplementedException();
     }
     
-    public void PmtEvent(ProgramMapping result, int pmtPid)
-    {
-        throw new NotImplementedException();
-    }
-
     public void OnBatBouquet(BatBouquet batBouquet)
     {
         throw new NotImplementedException();
@@ -170,4 +166,14 @@ public class Matenro : IPatEventHandler, ICatEventHandler, IPmtEventHandler, IBa
     {
         throw new NotImplementedException();
     }
+
+    public void PmtEvent(IsdbPmt result, int sourcePid)
+    {
+        throw new NotImplementedException();
+    }
+
+    public void PmtError(IsdbPmtErrors esInfoLengthLongerThanBuffer, int pmtPid, ushort programNumber)
+    {
+        throw new NotImplementedException();
+    }
 }

skyscraper8/Isdb/Parsers/IsdbPmtParser.cs

@@ -0,0 +1,156 @@
+using skyscraper5.Dvb.Descriptors;
+using skyscraper5.Mpeg2;
+using skyscraper5.Mpeg2.Descriptors;
+using skyscraper5.Mpeg2.Psi.Model;
+using skyscraper5.Skyscraper.IO;
+using skyscraper8.Isdb.Descriptors;
+using skyscraper8.Isdb.EventHandlers;
+using skyscraper8.Isdb.Tables;
+
+namespace skyscraper8.Isdb.Psi;
+
+public class IsdbPmtParser(IIsdbPmtEventHandler handler, int pmtPid, ushort programNumber) : IPsiProcessor
+{
+    public IIsdbPmtEventHandler Handler { get; } = handler;
+    public int PmtPid { get; } = pmtPid;
+    public ushort ProgramNumber { get; } = programNumber;
+
+    public void GatherPsi(PsiSection section, int sourcePid)
+    {
+        if (sourcePid != PmtPid)
+        {
+            Handler.PmtError(IsdbPmtErrors.SourcePidPmtPidMismatch,pmtPid,ProgramNumber);
+            return;
+        }
+        
+        MemoryStream buffer = new MemoryStream(section.GetDataCopy());
+
+        byte table_id = buffer.ReadUInt8();
+        if (table_id != 0x02)
+        {
+            Handler.PmtError(IsdbPmtErrors.WrongTableId, PmtPid, ProgramNumber);
+            return;
+        }
+
+        ushort readUInt16Be = buffer.ReadUInt16BE();
+        bool sectionSyntaxIndicator = (readUInt16Be & 0x8000) != 0;
+        if (!sectionSyntaxIndicator)
+        {
+            //According to T-REC-H.222.0-201808-S!!PDF-E.pdf, page 71 - this is supposed to be set to false.
+            Handler.PmtError(IsdbPmtErrors.SectionSyntaxIndicatorMissing, PmtPid, ProgramNumber);
+            return;
+        }
+        bool zero = (readUInt16Be & 0x4000) != 0;
+        int reserved = (readUInt16Be & 0x3000) >> 12;
+        int sectionLength = (readUInt16Be & 0x0fff);
+
+        ushort programNumber = buffer.ReadUInt16BE();
+        if (programNumber != ProgramNumber)
+        {
+            Handler.PmtError(IsdbPmtErrors.ProgramNumberMismatch, PmtPid, ProgramNumber);
+            return;
+        }
+
+        byte readUInt8 = buffer.ReadUInt8();
+        int reserved2 = (readUInt8 & 0xc0) >> 6;
+        int versionNumber = (readUInt8 & 0x3e);
+        bool currentNextIndicator = (readUInt8 & 0x01) != 0;
+
+        byte sectionNumber = buffer.ReadUInt8();
+
+        byte lastSectionNumber = buffer.ReadUInt8();
+
+        readUInt16Be = buffer.ReadUInt16BE();
+        int reserved3 = (readUInt16Be & 0xe000) >> 13;
+        int pcrPid = (readUInt16Be & 0x1fff);
+
+        readUInt16Be = buffer.ReadUInt16BE();
+        int reserved4 = (readUInt16Be & 0xf000) >> 12;
+        int programInfoLength = (readUInt16Be & 0x0fff) & 0x3ff;
+
+        IsdbPmt result = new IsdbPmt(programNumber, pcrPid);
+        if (programInfoLength > 0)
+        {
+            byte[] descriptorBuffer = new byte[programInfoLength];
+            int descriptorBufferFilling = buffer.Read(descriptorBuffer, 0, programInfoLength);
+            if (descriptorBufferFilling != programInfoLength)
+            {
+                Handler.PmtError(IsdbPmtErrors.DescriptorBufferLongerThanSourceBuffer,PmtPid,ProgramNumber);
+                return;
+            }
+
+            ParseOuterIsdbDescriptors(result, descriptorBuffer);
+        }
+
+        while ((buffer.Length - buffer.Position) > 4)
+        {
+            PmtStreamType streamType = (PmtStreamType)buffer.ReadUInt8();
+
+            readUInt16Be = buffer.ReadUInt16BE();
+            int reserved5 = (readUInt16Be & 0xe000) >> 13;
+            int elementaryPid = (readUInt16Be & 0x1fff);
+
+            readUInt16Be = buffer.ReadUInt16BE();
+            int reserved6 = (readUInt16Be & 0xf000) >> 12;
+            int esInfoLength = (readUInt16Be & 0xfff) & 0x3ff;
+            if (esInfoLength > buffer.GetAvailableBytes())
+            {
+                Handler.PmtError(IsdbPmtErrors.EsInfoLengthLongerThanBuffer, PmtPid, ProgramNumber);
+                return;
+            }
+
+            byte[] descriptorBuffer2 = buffer.ReadBytes(esInfoLength);
+            IsdbPmtStream streamChild = new IsdbPmtStream(streamType, elementaryPid);
+            result.AddStream(streamChild);
+            ParseInnerIsdbDescriptors(streamChild, descriptorBuffer2);
+        }
+
+        Handler?.PmtEvent(result, sourcePid);
+
+    }
+
+    private void ParseInnerIsdbDescriptors(IsdbPmtStream streamChild, byte[] descriptorBuffer2)
+    {
+        IEnumerable<TsDescriptor> descriptors = TsDescriptorUnpacker.GetInstance().UnpackDescriptors(descriptorBuffer2, "ISDB PMT");
+        foreach (TsDescriptor descriptor in descriptors)
+        {
+            switch (descriptor)
+            {
+                case StreamIdentifierDescriptor sid:
+                    if (streamChild.StreamIdentifierComponentTag == null)
+                        streamChild.StreamIdentifierComponentTag = sid.ComponentTag;
+                    else
+                        throw new NotImplementedException("Multiple stream identifier components are not yet supported - it would be great if you could share a sample recording of this stream so I can implement this.");
+                    break;
+                case Iso639LanguageDescriptor ild:
+                    if (streamChild.Iso639Language == null)
+                        streamChild.Iso639Language = ild;
+                    else
+                        throw new NotImplementedException("Multiple ISO639 Languages are not yet supported - it would be great if you could share a sample recording of this stream so I can implement this.");
+                    break;
+                case _0xFD_DataComponentDescriptor dcd:
+                    if (streamChild.DataComponent == null)
+                        streamChild.DataComponent = dcd;
+                    else
+                        throw new NotImplementedException("Multiple Data components are not yet supported - it would be great if you could share a sample recording of this stream so I can implement this.");
+                    break;
+                default:
+                    throw new NotImplementedException("Unknown descriptor type in ISDB PMT: " + descriptor.GetType() + " - it would be fabulous if you could share a sample of this stream so this can be implemented!");
+            }
+        }
+    }
+
+    private void ParseOuterIsdbDescriptors(IsdbPmt result, byte[] descriptorBuffer)
+    {
+        IEnumerable<TsDescriptor> descriptors = TsDescriptorUnpacker.GetInstance().UnpackDescriptors(descriptorBuffer, "ISDB PMT");
+        foreach (TsDescriptor descriptor in descriptors)
+        {
+            switch (descriptor)
+            {
+                default:
+                    throw new NotImplementedException("Unknown descriptor type in ISDB PMT: " + descriptor.GetType() + " - it would be fabulous if you could share a sample of this stream so this can be implemented!");
+            }
+        }
+
+    }
+}

skyscraper8/Isdb/Tables/IsdbPmt.cs

@@ -0,0 +1,17 @@
+namespace skyscraper8.Isdb.Tables;
+
+public class IsdbPmt(ushort programNumber, int pcrPid)
+{
+    public ushort ProgramNumber { get; } = programNumber;
+    public int PcrPid { get; } = pcrPid;
+
+    public List<IsdbPmtStream> Streams;
+    public void AddStream(IsdbPmtStream streamChild)
+    {
+        if (Streams == null)
+            Streams = new List<IsdbPmtStream>();
+        
+        Streams.Add(streamChild);
+        
+    }
+}

skyscraper8/Isdb/Tables/IsdbPmtStream.cs

@@ -0,0 +1,14 @@
+using skyscraper5.Mpeg2.Descriptors;
+using skyscraper5.Mpeg2.Psi.Model;
+using skyscraper8.Isdb.Descriptors;
+
+namespace skyscraper8.Isdb.Tables;
+
+public class IsdbPmtStream(PmtStreamType streamType, int elementaryPid)
+{
+    public PmtStreamType StreamType { get; } = streamType;
+    public int ElementaryPid { get; } = elementaryPid;
+    public byte? StreamIdentifierComponentTag { get; set; }
+    public Iso639LanguageDescriptor Iso639Language { get; set; }
+    public _0xFD_DataComponentDescriptor DataComponent { get; set; }
+}

skyscraper8/Mpeg2/Descriptors/0x0A_Iso639LanguageDescriptor.cs

@@ -5,9 +5,9 @@ using skyscraper5.Skyscraper.Plugins;
 namespace skyscraper5.Mpeg2.Descriptors
 {
 	[SkyscraperPlugin]
-	[TsDescriptor(10, "PMT")]
+	[TsDescriptor(10, "PMT", "ISDB PMT")]
     [BannedTable("TSDT","SDT")]
-    class Iso639LanguageDescriptor : TsDescriptor
+    public class Iso639LanguageDescriptor : TsDescriptor
     {
         public Iso639LanguageDescriptor(byte[] buffer)
         {

skyscraper8/Mpeg2/Psi/PmtParser.cs

@@ -17,7 +17,7 @@ using skyscraper8.Skyscraper;
 
 namespace skyscraper5.Mpeg2.Psi
 {
-    class PmtParser : IPsiProcessor, IMultistandardTable
+    class PmtParser : IPsiProcessor
     {
         public IPmtEventHandler EventHandler { get; }
 
@@ -100,7 +100,7 @@ namespace skyscraper5.Mpeg2.Psi
 
         private void ParseInnerDescriptors(ProgramMappingStream output, byte[] inputBuffer)
         {
-            IEnumerable<TsDescriptor> descriptors = TsDescriptorUnpacker.GetInstance().UnpackDescriptors(inputBuffer, GetTableType());
+            IEnumerable<TsDescriptor> descriptors = TsDescriptorUnpacker.GetInstance().UnpackDescriptors(inputBuffer, "PMT");
             foreach (TsDescriptor dvbDescriptor in descriptors)
             {
                 output.NumDescriptors++;
@@ -408,7 +408,7 @@ namespace skyscraper5.Mpeg2.Psi
 
         private void ParseOuterDescriptors(ProgramMapping output, byte[] inputBuffer)
         {
-            IEnumerable<TsDescriptor> descriptors = TsDescriptorUnpacker.GetInstance().UnpackDescriptors(inputBuffer, GetTableType());
+            IEnumerable<TsDescriptor> descriptors = TsDescriptorUnpacker.GetInstance().UnpackDescriptors(inputBuffer, "PMT");
             foreach (TsDescriptor tsDescriptor in descriptors)
             {
                 string name = tsDescriptor.GetType().Name;
@@ -588,26 +588,6 @@ namespace skyscraper5.Mpeg2.Psi
                     default:
                         throw new NotImplementedException(name);
                 }
-                
-            }
-
-        }
-
-        private STD_TYPE standard;
-        public void SetStandard(STD_TYPE standard)
-        {
-            this.standard = standard;
-        }
-
-        private string GetTableType()
-        {
-            if (IMultistandardTable.IsIsdb(standard))
-            {
-                return "ISDB PMT";
-            }
-            else
-            {
-                return "PMT";
             }
         }
     }

skyscraper8/Mpeg2/TsDescriptorUnpacker.cs

@@ -31,18 +31,39 @@ namespace skyscraper5.Mpeg2
                 throw new AccessViolationException("Wait, this isn't supposed to happen! There are no descriptor types in my assembly!");
             }
             
-            userPrivates = new bool[byte.MaxValue + 1];
+            //Set DVB user privates
+            dvbUserPrivates = new bool[byte.MaxValue + 1];
+            for (byte i = 0x80; i < 0xfe; i++)
+            {
+                dvbUserPrivates[i] = true;
+            }
+            dvbUserPrivates[0xfe] = true;
+            
+            //Set ISDB user privates
+            isdbUserPrivates = new bool[byte.MaxValue + 1];
+            for (byte i = 0x80; i < 0xbf; i++)
+            {
+                isdbUserPrivates[i] = true;
+            }
+            isdbUserPrivates[0xbf] = true;
         }
         
+        private bool[] dvbUserPrivates;
+        private bool[] isdbUserPrivates;
 
-
-        private bool[] userPrivates;
-
-        public void SetUserDefined(byte descriptorId, bool userDefined)
+        private bool IsUserPrivate(string tableName, byte tag)
         {
-            userPrivates[descriptorId] = userDefined;
+            if (tableName.Contains("ISDB"))
+            {
+                return isdbUserPrivates[tag];
+            }
+            else
+            {
+                return dvbUserPrivates[tag];
+            }
         }
 
+
         public static TsDescriptorUnpacker GetInstance()
         {
             if (_instance == null)
@@ -72,10 +93,15 @@ namespace skyscraper5.Mpeg2
                 }
 
                 byte[] descriptorContent = ms.ReadBytes(descriptorLength);
+
+                if (!descriptorMap.ContainsKey(callingTable))
+                {
+                    throw new Mpeg2Exception(String.Format("Failed to load descriptors for table {0} in main assembly. The first descriptor has ID 0x{1:X2} ({1}) and is {2} bytes long.", callingTable,descriptorTag,descriptorLength));
+                }
                 ConstructorInfo[] descriptorAttributes = descriptorMap[callingTable];
                 if (descriptorAttributes[descriptorTag] == null)
                 {
-					if (userPrivates[descriptorTag])
+					if (IsUserPrivate(callingTable, descriptorTag))
 					{
 						yield return new UserDefinedDescriptor(descriptorTag, descriptorContent);
 						continue;

skyscraper8/Skyscraper/IMultistandardTable.cs

@@ -29,6 +29,11 @@ public interface IMultistandardTable
             case STD_TYPE.STD_DVBT:
             case STD_TYPE.STD_DVBT2:
                 return true;
+            case STD_TYPE.STD_AUTO:
+                //Since skyscraper8 was meant to be designed around DVB, the older code which is non-multistandard-aware
+                //will never touch SetStandard(...), therefore leaving it at it's default value, which is "STD_AUTO"  - 
+                //so we can safely assume that AUTO = DVB
+                return true;
             default:
                 return false;
         }

skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs

@@ -133,12 +133,7 @@ namespace skyscraper5.Skyscraper.Scraper
         public SkyscraperContext(TsContext dvbContext, DataStorage dataStorage = null, ObjectStorage objectStorage = null, int skipAtStart = 0, int? currentNetworkId = null, int? currentTransportStreamId = null)
         {
             TsDescriptorUnpacker descriptorUnpacker = TsDescriptorUnpacker.GetInstance();
-            for (byte i = 0x80; i < 0xfe; i++)
-            {
-                descriptorUnpacker.SetUserDefined(i, true);
-            }
             
-            descriptorUnpacker.SetUserDefined(0xfe, true);
 
             DvbContext = dvbContext;
             DvbContext.OnBadReception = (() => 

skyscraper8/Skyscraper/Scraper/StreamAutodetection/StreamTypeAutodetectionTest.cs

@@ -14,10 +14,6 @@ namespace skyscraper5.Skyscraper.Scraper.StreamAutodetection
         public StreamTypeAutodetectionTest()
         {
             TsDescriptorUnpacker descriptorUnpacker = TsDescriptorUnpacker.GetInstance();
-            for (byte i = 0x80; i < 0xfe; i++)
-            {
-                descriptorUnpacker.SetUserDefined(i, true);
-            }
         }
 
         public void PushPacket(byte[] buffer)