Began working on ISDB subtitles.

Documentation/AI Food/ISDB Subtitle Question.txt

@@ -0,0 +1,18 @@
+I'm trying to parse the subtitles from a Brazilian live ISDB-T Capture.
+I parsed the PMT, and it had a Data Component Descriptor present for the corresponding PID.
+The Data Component Descriptor said 0x0012 for it's data_component_id.
+According to Annex J in ARIB STD-B10, Part 3 this means "Subtitle coding for digital terrestrial broadcasting (C profile)" and points to ARIB STD-B24, Part 3, Volume 1.
+So I checked ARIB STD-B24 Version 6.4 Fascicle 1, Part 3, Chapter 9.
+This states that the captions are transmitted in PES packets, so I coded my program to attach a PES listener to the corresponding PID. 
+As expected, I get valid PES packets from it.
+But the structure from the payload does not match what I find in ARIB STD-B24 Version 6.4 Fascicle 1, Part 3, Clause 9.2
+The first bytes I got are 0x80 0xFF 0xF0 0x04 0x00 0x00 0x00 0x87
+Bytes 1 & 2 in the standard say that they're data_group_link_number and last_data_group_link_number.
+But that doesn't fit. 0x0F0 would be the last data group link number, but it's smaller than 0xFF.
+I think I am looking in the wrong spot. Could you give me a small push in the right direction?
+
+-
+
+Okay, I think I got a few bytes.
+The first three bytes 0x80, 0xFF, 0xF0 can be explained using ARIB TR-B14 Fascicle 3, clause 6.2.2 and ARIB STD-B37, clause 2.2.3.6
+These are indeed a data_identifier, private_stream_id, reserved bits, and pes_data_packet_header_length, but now I still need to figure out the next four bytes: 0x00, 0x00, 0x00, 0x87. The 0x87 seems to indeed be the length of the packet.

Documentation/ImHex Patterns/ISDB Caption PES.hexpat

@@ -0,0 +1,28 @@
+#pragma endian big
+
+bitfield thirdByte
+{
+    reserved : 4;
+    pes_data_packet_header_length : 4;
+};
+
+bitfield DataGroupHeader
+{
+    data_group_id : 6;
+    data_group_version : 2;
+    data_group_link_number : 8;
+    last_data_group_link_number : 8;
+};
+
+struct DataGroup
+{
+    u8 data_identifier;
+    u8 private_stream_id;
+    thirdByte thirdbyte;
+    DataGroupHeader header_bitfield;
+    u16 data_group_size;
+    u8 data_group_byte[data_group_size];
+    u16 crc16;
+};
+
+DataGroup header_bitfield @ 0x0000;

skyscraper8/Isdb/Matenro.cs

@@ -9,6 +9,7 @@ using skyscraper5.Mpeg2.Psi;
 using skyscraper5.Mpeg2.Psi.Model;
 using skyscraper5.Skyscraper.IO.CrazycatStreamReader;
 using skyscraper5.Skyscraper.Scraper;
+using skyscraper5.Skyscraper.Scraper.StreamAutodetection;
 using skyscraper8.Isdb.EventHandlers;
 using skyscraper8.Isdb.Psi;
 using skyscraper8.Isdb.Subtitles;
@@ -17,7 +18,7 @@ using skyscraper8.Skyscraper.Scraper.Storage;
 
 namespace skyscraper8.Isdb;
 
-public class Matenro : IPatEventHandler, ICatEventHandler, IBatEventHandler, ISdtEventHandler, IEitEventHandler, IRstEventHandler, ITdtEventHandler, ITotEventHandler, IIsdbPmtEventHandler, IBitEventHandler, IIsdbSubtitleHandler
+public class Matenro : IPatEventHandler, ICatEventHandler, IBatEventHandler, ISdtEventHandler, IEitEventHandler, IRstEventHandler, ITdtEventHandler, ITotEventHandler, IIsdbPmtEventHandler, IBitEventHandler, IIsdbSubtitleHandler, IAutodetectionEventHandler
 {
     public TsContext DvbContext { get; }
     public DataStorage DataStorage { get; }
@@ -206,7 +207,7 @@ public class Matenro : IPatEventHandler, ICatEventHandler, IBatEventHandler, ISd
         {
             if (pidsReferencedByPmt == null)
                 pidsReferencedByPmt = new bool[0x2000];
-            if (pidsReferencedByPmt[sourcePid])
+            if (pidsReferencedByPmt[pmtStream.ElementaryPid])
                 continue;
             
             pidsReferencedByPmt[sourcePid] = true;
@@ -221,6 +222,9 @@ public class Matenro : IPatEventHandler, ICatEventHandler, IBatEventHandler, ISd
                 case StreamType.IsdbTeletext:
                     DvbContext.RegisterPacketProcessor(pmtStream.ElementaryPid, new PesDecoder(new IsdbSubtitlePesParser(this)));
                     continue;
+                case StreamType.TryAutodetect:
+                    DvbContext.RegisterPacketProcessor(pmtStream.ElementaryPid, new StreamTypeAutodetection(pmtStream.ElementaryPid, this));
+                    continue;
                 default:
                     throw new NotImplementedException(String.Format("Could not figure out what to do with a {0} stream. It would be great if you could share a sample of this stream, so I can try to implement this.", streamType));
             }
@@ -256,7 +260,9 @@ public class Matenro : IPatEventHandler, ICatEventHandler, IBatEventHandler, ISd
             return StreamType.Audio;
         if (pmtStream.StreamType == PmtStreamType.Iso11172Video)
             return StreamType.Video;
-        
+        if (pmtStream.StreamType == PmtStreamType.Iso13818_1PesPackets && pmtStream.NumDescriptors == 1 && pmtStream.DataStreamAlignmentType != null)
+            return StreamType.TryAutodetect;
+            
         throw new IsdbException(String.Format("Could not detect stream type for PID 0x{0:X4}. It would be great if you could share a sample of this stream, so I can try to implement this.", pmtStream.ElementaryPid));
     }
 
@@ -289,4 +295,14 @@ public class Matenro : IPatEventHandler, ICatEventHandler, IBatEventHandler, ISd
             bits[bit.OriginalNetworkId] = true;
         }
     }
+
+    public void AutodetectionRuleOut(int pid, Contestant contestant, ProgramContext programContext)
+    {
+        throw new NotImplementedException();
+    }
+
+    public void AutodetectionSucessful(int pid, Contestant contestant, ProgramContext programContext)
+    {
+        throw new NotImplementedException();
+    }
 }

skyscraper8/Isdb/Parsers/IsdbPmtParser.cs

@@ -114,6 +114,7 @@ public class IsdbPmtParser(IIsdbPmtEventHandler handler, int pmtPid, ushort prog
         IEnumerable<TsDescriptor> descriptors = TsDescriptorUnpacker.GetInstance().UnpackDescriptors(descriptorBuffer2, "ISDB PMT");
         foreach (TsDescriptor descriptor in descriptors)
         {
+            streamChild.NumDescriptors++;
             switch (descriptor)
             {
                 case StreamIdentifierDescriptor sid:

skyscraper8/Isdb/Subtitles/IsdbSubtitlePesParser.cs

@@ -1,3 +1,4 @@
+using log4net;
 using skyscraper5.Mpeg2;
 
 namespace skyscraper8.Isdb.Subtitles;
@@ -5,6 +6,8 @@ namespace skyscraper8.Isdb.Subtitles;
 class IsdbSubtitlePesParser : IPesProcessor
 {
     public IIsdbSubtitleHandler Handler { get; }
+    private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
+
 
     public IsdbSubtitlePesParser(IIsdbSubtitleHandler handler)
     {

skyscraper8/Isdb/Tables/IsdbPmtStream.cs

@@ -16,4 +16,5 @@ public class IsdbPmtStream(PmtStreamType streamType, int elementaryPid)
     public byte? DataStreamAlignmentType { get; set; }
     public AvcVideoDescriptor AvcVideo { get; set; }
     public AacDescriptor Aac { get; set; }
+    public int NumDescriptors { get; set; }
 }