ft/skyscraper8
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Got ISDB BIT working for first Brazilian sample.
All checks were successful
🚀 Pack skyscraper8 / make-zip (push) Successful in 52s
Details

Browse Source
This commit is contained in:
fey 2026-08-30 00:04:05 +02:00
parent f5b8a76803
commit 252abb6330
15 changed files with 439 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Gimmicks/skyscraper5.LyngsatMapsScraper/Program.cs
View File

@ -9,6 +9,7 @@ using skyscraper5.LyngsatMapsScraper.Model;
using skyscraper5.Skyscraper.Scraper.Storage;
using skyscraper8.Skyscraper.Plugins;
using skyscraper8.Skyscraper.Scraper.Storage;
using TimeSpan = System.TimeSpan;
namespace skyscraper5.LyngsatMapsScraper
{
@ -167,5 +168,20 @@ namespace skyscraper5.LyngsatMapsScraper
{
return File.ReadAllText(fileInfo.FullName);
}
public static TimeSpan GetAge(this FileInfo fileInfo)
{
DateTime lastWriteTime = fileInfo.LastWriteTime;
DateTime now = DateTime.Now;
TimeSpan diff = now - lastWriteTime;
return diff;
}
public static int GetAgeInSeconds(this FileInfo fileInfo)
{
TimeSpan timeSpan = fileInfo.GetAge();
int result = (int)timeSpan.TotalSeconds;
return result;
}
}
}

87
skyscraper8/Isdb/Descriptors/0xCE_ExtendedBroadcasterDescriptor.cs Normal file
View File

@ -0,0 +1,87 @@
using log4net;
using skyscraper5.Mpeg2;
using skyscraper5.Skyscraper.IO;
using skyscraper5.Skyscraper.Plugins;
namespace skyscraper8.Isdb.Descriptors;
[SkyscraperPlugin]
[TsDescriptor(0xce,"BIT")]
public class _0xCE_ExtendedBroadcasterDescriptor : TsDescriptor {
private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
public _0xCE_ExtendedBroadcasterDescriptor(byte[] buffer)
{
MemoryStream ms = new MemoryStream(buffer);
byte braodcasterType = ms.ReadUInt8();
braodcasterType >>= 4;
BroadcasterType = braodcasterType;
switch (braodcasterType)
{
case 0x01: //Terristial broadcaster
TerristialBroadcasterId = ms.ReadUInt16BE();
byte numbers = ms.ReadUInt8();
int numberOfAffiliationIdLoop = (numbers & 0xf0);
numberOfAffiliationIdLoop >>= 4;
int numberOfBroadcasterIdLoop = (numbers & 0x0f);
Affiliations = ms.ReadBytes(numberOfAffiliationIdLoop);
OriginalNetworkIds = new ushort[numberOfBroadcasterIdLoop];
BroadcasterIds = new byte[numberOfBroadcasterIdLoop];
for (int j = 0; j < numberOfBroadcasterIdLoop; j++)
{
OriginalNetworkIds[j] = ms.ReadUInt16BE();
BroadcasterIds[j] = ms.ReadUInt8();
}
PrivateDataBytes = ms.ReadBytes(ms.GetAvailableBytes());
Valid = true;
break;
case 0x02:
TerristialBroadcasterId = ms.ReadUInt16BE();
byte numbersB = ms.ReadUInt8();
int numberOfAffiliationIdLoopB = (numbersB & 0xf0);
numberOfAffiliationIdLoopB >>= 4;
int numberOfBroadcasterIdLoopB = (numbersB & 0x0f);
Affiliations = ms.ReadBytes(numberOfAffiliationIdLoopB);
OriginalNetworkIds = new ushort[numberOfBroadcasterIdLoopB];
BroadcasterIds = new byte[numberOfBroadcasterIdLoopB];
for (int j = 0; j < numberOfBroadcasterIdLoopB; j++)
{
OriginalNetworkIds[j] = ms.ReadUInt16BE();
BroadcasterIds[j] = ms.ReadUInt8();
}
PrivateDataBytes = ms.ReadBytes(ms.GetAvailableBytes());
Valid = true;
break;
default:
logger.Error($"Unknown broadcaster type in ISDB Broadcaster Descriptor: {braodcasterType}");
Valid = false;
break;
}
}
public byte[] PrivateDataBytes { get; set; }
public byte[] BroadcasterIds { get; set; }
public ushort[] OriginalNetworkIds { get; set; }
public byte[] Affiliations { get; set; }
public ushort TerristialBroadcasterId { get; set; }
public byte BroadcasterType { get; set; }
}

17
skyscraper8/Isdb/Descriptors/0xD8_BroadcasterNameDescriptor.cs Normal file
View File

@ -0,0 +1,17 @@
using System.Text;
using skyscraper5.Mpeg2;
using skyscraper5.Skyscraper.Plugins;
using skyscraper8.Skyscraper.Text;
namespace skyscraper8.Isdb.Descriptors;
[SkyscraperPlugin]
[TsDescriptor(0xd8,"BIT")]
public class _0xD8_BroadcasterNameDescriptor : TsDescriptor {
public _0xD8_BroadcasterNameDescriptor(byte[] buffer)
{
BroadcasterName = AribStdB24TextDecoder.GetInstance().Decode(buffer);
}
public string BroadcasterName { get; set; }
}

40
skyscraper8/Isdb/Descriptors/0xDA_SIprimeTSdescriptor.cs Normal file
View File

@ -0,0 +1,40 @@
using skyscraper5.Mpeg2;
using skyscraper5.Skyscraper.IO;
using skyscraper5.Skyscraper.Plugins;
namespace skyscraper8.Isdb.Descriptors;
[SkyscraperPlugin]
[TsDescriptor(0xDA,"BIT")]
public class _0xDA_SIprimeTSdescriptor : TsDescriptor {
public _0xDA_SIprimeTSdescriptor(byte[] buffer)
{
MemoryStream ms = new MemoryStream(buffer, false);
ParameterVersion = ms.ReadUInt8();
UpdateTime = ms.ReadLower16bitsOfMjd();
SiPrimeTsNetworkId = ms.ReadUInt16BE();
SiPrimeTsId = ms.ReadUInt16BE();
while (ms.GetAvailableBytes() >= 2)
{
byte tableId = ms.ReadUInt8();
byte tableDescriptionLength = ms.ReadUInt8();
byte[] tableDescription = ms.ReadBytes(tableDescriptionLength);
if (_tableDescriptions == null)
_tableDescriptions = new Dictionary<byte, byte[]>();
_tableDescriptions.Add(tableId, tableDescription);
}
Valid = true;
}
private Dictionary<byte, byte[]> _tableDescriptions;
public ushort SiPrimeTsId { get; set; }
public ushort SiPrimeTsNetworkId { get; set; }
public DateTime? UpdateTime { get; set; }
public byte ParameterVersion { get; set; }
}

7
skyscraper8/Isdb/EventHandlers/IBitEventHandler.cs
View File

@ -1,13 +1,18 @@
using skyscraper8.Isdb.Tables;
namespace skyscraper8.Isdb.EventHandlers;
public interface IBitEventHandler
{
void BitError(BitErrors error);
void OnBit(Bit bit);
}
public enum BitErrors
{
WrongTableId,
SyntaxIndicatorMissing,
SectionLengthWrong
SectionLengthWrong,
FirstDescriptorLengthTooLarge,
BroadcasterDescriptorLengthTooLarge
}

38
skyscraper8/Isdb/Matenro.cs
View File

@ -17,7 +17,7 @@ using skyscraper8.Skyscraper.Scraper.Storage;
namespace skyscraper8.Isdb;
public class Matenro : IPatEventHandler, ICatEventHandler, IBatEventHandler, ISdtEventHandler, IEitEventHandler, IRstEventHandler, ITdtEventHandler, ITotEventHandler, IIsdbPmtEventHandler
public class Matenro : IPatEventHandler, ICatEventHandler, IBatEventHandler, ISdtEventHandler, IEitEventHandler, IRstEventHandler, ITdtEventHandler, ITotEventHandler, IIsdbPmtEventHandler, IBitEventHandler, IIsdbSubtitleHandler
{
public TsContext DvbContext { get; }
public DataStorage DataStorage { get; }
@ -72,7 +72,7 @@ public class Matenro : IPatEventHandler, ICatEventHandler, IBatEventHandler, ISd
dvbContext.RegisterPacketProcessor(0x0021, new PsiDecoder(0x0021, _ertParser));
dvbContext.RegisterPacketProcessor(0x0022, new PsiDecoder(0x0022, new PcatParser()));
dvbContext.RegisterPacketProcessor(0x0023, new PsiDecoder(0x0023, sdttParser));
dvbContext.RegisterPacketProcessor(0x0024, new PsiDecoder(0x0024, new BitParser()));
dvbContext.RegisterPacketProcessor(0x0024, new PsiDecoder(0x0024, new BitParser(this)));
dvbContext.RegisterPacketProcessor(0x0025, new PsiDecoder(0x0025, new IsdbPid25Parser()));
dvbContext.RegisterPacketProcessor(0x0026, new PsiDecoder(0x0026, eitParser));
dvbContext.RegisterPacketProcessor(0x0027, new PsiDecoder(0x0027, eitParser));
@ -116,15 +116,18 @@ public class Matenro : IPatEventHandler, ICatEventHandler, IBatEventHandler, ISd
public void SetNetworkId(ushort networkId)
{
throw new NotImplementedException();
this._networkId = networkId;
}
public void SetTransportStreamId(ushort transportStreamId)
{
if (_transportStreamId != null)
{
if (transportStreamId != _transportStreamId)
{
logger.InfoFormat("TSID changed from {0} to {1}", this._transportStreamId.Value, transportStreamId);
}
}
_transportStreamId = transportStreamId;
}
@ -189,7 +192,7 @@ public class Matenro : IPatEventHandler, ICatEventHandler, IBatEventHandler, ISd
if (!storedPmts.Contains(hash))
{
bool inserted = false;
if (DataStorage.TestForIsdbPmt(_networkId.Value, _transportStreamId.Value, result.ProgramNumber))
if (!DataStorage.TestForIsdbPmt(_networkId.Value, _transportStreamId.Value, result.ProgramNumber))
{
DataStorage.InsertIsdbPmt(_networkId.Value, _transportStreamId.Value, result);
inserted = true;
@ -216,7 +219,7 @@ public class Matenro : IPatEventHandler, ICatEventHandler, IBatEventHandler, ISd
continue;
case StreamType.IsdbSubtitles:
case StreamType.IsdbTeletext:
DvbContext.RegisterPacketProcessor(pmtStream.ElementaryPid, new PesDecoder(new IsdbSubtitlePesParser()));
DvbContext.RegisterPacketProcessor(pmtStream.ElementaryPid, new PesDecoder(new IsdbSubtitlePesParser(this)));
continue;
default:
throw new NotImplementedException(String.Format("Could not figure out what to do with a {0} stream. It would be great if you could share a sample of this stream, so I can try to implement this.", streamType));
@ -261,4 +264,29 @@ public class Matenro : IPatEventHandler, ICatEventHandler, IBatEventHandler, ISd
{
throw new NotImplementedException();
}
private bool bitError;
public void BitError(BitErrors error)
{
if (!bitError)
{
logger.ErrorFormat("Invalid ISDB BIT: {0}", error.ToString());
bitError = true;
}
}
private bool[] bits;
public void OnBit(Bit bit)
{
SetNetworkId(bit.OriginalNetworkId);
if (bits == null)
bits = new bool[0x2000];
if (!bits[bit.OriginalNetworkId])
{
DataStorage.UpsertBit(bit);
logger.InfoFormat("ISDB Broadcaster ID: {0:X4}", bit.OriginalNetworkId);
bits[bit.OriginalNetworkId] = true;
}
}
}

94
skyscraper8/Isdb/Parsers/BitParser.cs
View File

@ -1,12 +1,21 @@
using moe.yo3explorer.skyscraper8.DVBI.Model;
using skyscraper5.Mpeg2;
using skyscraper5.Skyscraper.IO;
using skyscraper8.Isdb.Descriptors;
using skyscraper8.Isdb.EventHandlers;
using skyscraper8.Isdb.Tables;
namespace skyscraper8.Isdb.Psi;
public class BitParser : IPsiProcessor
{
private IBitEventHandler eventHandler;
public BitParser(IBitEventHandler eventHandler)
{
this.eventHandler = eventHandler;
}
public void GatherPsi(PsiSection section, int sourcePid)
{
byte[] data = section.GetDataCopy();
@ -36,7 +45,90 @@ public class BitParser : IPsiProcessor
//continue on 6-STD-B10v5_13-E1.pdf, page 123
ushort originalNetworkId = ms.ReadUInt16BE();
throw new NotImplementedException();
byte byteA = ms.ReadUInt8();
int versionNumber = (byteA & 0x3e) >> 1;
bool currentNextIndicator = (byteA & 0x01) != 0;
byte sectionNumber = ms.ReadUInt8();
byte lastSectionNumber = ms.ReadUInt8();
ushort firstDescriptorsLength = ms.ReadUInt16BE();
bool broadcastViewProperty = (firstDescriptorsLength & 0x1000) != 0;
firstDescriptorsLength &= 0x0fff;
if (firstDescriptorsLength > ms.GetAvailableBytes())
{
eventHandler.BitError(BitErrors.FirstDescriptorLengthTooLarge);
return;
}
byte[] firstDescriptorsBuffer = ms.ReadBytes(firstDescriptorsLength);
Bit bit = new Bit(originalNetworkId, broadcastViewProperty);
ParseFirstDescriptors(firstDescriptorsBuffer, bit);
while (ms.GetAvailableBytes() > 4)
{
byte broadcasterId = ms.ReadUInt8();
ushort broadcasterDescriptorsLength = ms.ReadUInt16BE();
broadcasterDescriptorsLength &= 0x0fff;
if (broadcasterDescriptorsLength > ms.GetAvailableBytes())
{
eventHandler.BitError(BitErrors.BroadcasterDescriptorLengthTooLarge);
return;
}
BitBroadcaster broadcaster = new BitBroadcaster();
byte[] broadcasterBuffer = ms.ReadBytes(broadcasterDescriptorsLength);
ParseInnerDescriptors(broadcasterBuffer, broadcaster);
if (bit.Broadcasters == null)
{
bit.Broadcasters = new Dictionary<byte, BitBroadcaster>();
}
bit.Broadcasters.Add(broadcasterId, broadcaster);
}
eventHandler.OnBit(bit);
}
private void ParseInnerDescriptors(byte[] broadcasterBuffer, BitBroadcaster broadcaster)
{
IEnumerable<TsDescriptor> descriptors = TsDescriptorUnpacker.GetInstance().UnpackDescriptors(broadcasterBuffer, "BIT");
foreach (TsDescriptor descriptor in descriptors)
{
switch (descriptor)
{
case _0xCE_ExtendedBroadcasterDescriptor ebd:
broadcaster.ExtendedBroadcaster = ebd;
break;
default:
throw new NotImplementedException(String.Format("Unknown descriptor type {0} in ISDB BIT. If possible, please share a sample recording, so this can be implemented.", descriptor.GetType()));
}
}
}
private void ParseFirstDescriptors(byte[] firstDescriptorsBuffer, Bit bit)
{
IEnumerable<TsDescriptor> descriptors = TsDescriptorUnpacker.GetInstance().UnpackDescriptors(firstDescriptorsBuffer, "BIT");
foreach (TsDescriptor descriptor in descriptors)
{
switch (descriptor)
{
case _0xD8_BroadcasterNameDescriptor bnd:
if (string.IsNullOrEmpty(bit.BroadcasterName))
bit.BroadcasterName = bnd.BroadcasterName;
else
throw new NotImplementedException("Multiple BroadcasterNameDescriptors not implemented yet - it would be great if you could share a sample of this stream, so this can be analyzed.");
break;
case _0xDA_SIprimeTSdescriptor siptsd:
if (bit.SiPrimeTs == null)
bit.SiPrimeTs = siptsd;
else
throw new NotImplementedException("Multiple SI Prime TS descriptors not yet implemented. Please share a sample of this stream, so this can be analyzed.");
break;
default:
throw new NotImplementedException(String.Format("Unknown descriptor type {0} in ISDB BIT. If possible, please share a sample recording, so this can be implemented.", descriptor.GetType()));
}
}
}
}

7
skyscraper8/Isdb/Subtitles/IsdbSubtitlePesParser.cs
View File

@ -4,6 +4,13 @@ namespace skyscraper8.Isdb.Subtitles;
class IsdbSubtitlePesParser : IPesProcessor
{
public IIsdbSubtitleHandler Handler { get; }
public IsdbSubtitlePesParser(IIsdbSubtitleHandler handler)
{
Handler = handler;
}
public void ProcessPesPacket(PesPacket packet)
{
throw new NotImplementedException();

23
skyscraper8/Isdb/Tables/Bit.cs Normal file
View File

@ -0,0 +1,23 @@
using skyscraper8.Isdb.Descriptors;
namespace skyscraper8.Isdb.Tables;
public class Bit
{
public ushort OriginalNetworkId { get; }
public bool BroadcastViewProperty { get; }
public string? BroadcasterName { get; set; }
public _0xDA_SIprimeTSdescriptor SiPrimeTs { get; set; }
public Dictionary<byte, BitBroadcaster> Broadcasters { get; set; }
public Bit(ushort originalNetworkId, bool broadcastViewProperty)
{
OriginalNetworkId = originalNetworkId;
BroadcastViewProperty = broadcastViewProperty;
}
}
public class BitBroadcaster
{
public _0xCE_ExtendedBroadcasterDescriptor ExtendedBroadcaster { get; set; }
}

23
skyscraper8/Skyscraper/IO/En300468AnnexCDateTime.cs
View File

@ -9,6 +9,7 @@ namespace skyscraper5.Skyscraper.IO
{
return ReadDateTime(s.ReadBytes(5));
}
public static DateTime? ReadDateTime(byte[] buffer)
{
if (buffer.Length != 5)
@ -43,6 +44,28 @@ namespace skyscraper5.Skyscraper.IO
return new DateTime(y, m, d, hour, minute, second);
}
public static DateTime? ReadLower16bitsOfMjd(this Stream s)
{
byte[] buffer = s.ReadBytes(2);
(buffer[1], buffer[0]) = (buffer[0], buffer[1]);
int mjd = BitConverter.ToUInt16(buffer, 0);
int y1 = (int)(((double)mjd - 15078.2) / 365.25);
int m1 = (int)((mjd - 14956.1 - (int)((double)y1 * 365.25)) / 30.6001);
int d = mjd - 14956 - (int)((double)y1 * 365.25) - (int)((double)m1 * 30.6001);
int k = 0;
if (m1 == 14 || m1 == 15)
k = 1;
int y = y1 + k + 1900;
int m = m1 - 1 - k * 12;
m1--;
if (y1 < 0)
return null;
return new DateTime(y, m, d);
}
/*
*

34
skyscraper8/Skyscraper/IO/FileInfoExtensions.cs Normal file
View File

@ -0,0 +1,34 @@
namespace skyscraper8.Skyscraper.IO;
public static class FileInfoExtensions
{
public static void WriteAllBytes(this FileInfo fileInfo, byte[] bytes)
{
File.WriteAllBytes(fileInfo.FullName, bytes);
}
public static byte[] ReadAllBytes(this FileInfo fileInfo)
{
return File.ReadAllBytes(fileInfo.FullName);
}
public static string ReadAllText(this FileInfo fileInfo)
{
return File.ReadAllText(fileInfo.FullName);
}
public static TimeSpan GetAge(this FileInfo fileInfo)
{
DateTime lastWriteTime = fileInfo.LastWriteTime;
DateTime now = DateTime.Now;
TimeSpan diff = now - lastWriteTime;
return diff;
}
public static int GetAgeInSeconds(this FileInfo fileInfo)
{
TimeSpan timeSpan = fileInfo.GetAge();
int result = (int)timeSpan.TotalSeconds;
return result;
}
}

6
skyscraper8/Skyscraper/Scraper/Storage/DataStorage.cs
View File

@ -215,5 +215,11 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
bool TestForIsdbPmt(ushort networkPid, ushort transportStreamId, ushort resultProgramNumber);
void InsertIsdbPmt(ushort networkPid, ushort transportStreamId, IsdbPmt result);
/// <summary>
/// Inserts or Updates an ISDB BIT table
/// </summary>
/// <param name="bit"></param>
void UpsertBit(Bit bit);
}
}

16
skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs
View File

@ -48,6 +48,7 @@ using System.Text;
using System.Text.RegularExpressions;
using skyscraper8.Atsc.A331.Schema;
using skyscraper8.Isdb.Tables;
using skyscraper8.Skyscraper.IO;
using skyscraper8.Skyscraper.Scraper.Storage.Tar;
using Platform = skyscraper5.Dvb.SystemSoftwareUpdate.Model.Platform;
@ -1812,6 +1813,21 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
File.WriteAllText(fi.FullName, JsonConvert.SerializeObject(result, Formatting.Indented, jsonSerializerSettings));
}
public void UpsertBit(Bit bit)
{
string combine = Path.Combine(rootDirectory.FullName, "BIT", bit.OriginalNetworkId.ToString() + ".json");
FileInfo fi = new FileInfo(combine);
if (fi.Exists)
{
if (fi.GetAgeInSeconds() < 60)
{
return;
}
}
EnsureDirectoryExists(fi.Directory);
File.WriteAllText(fi.FullName, JsonConvert.SerializeObject(bit, jsonSerializerSettings));
}
public RfSpectrumData GetRfSpectrum(Guid jobGuid)
{
string filename = Path.Combine(rootDirectory.FullName, "rf", jobGuid.ToString() + ".rf");

8
skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs
View File

@ -1951,5 +1951,13 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.InMemory
_isdbPmts.Add(key, result);
}
private Bit[] _bits;
public void UpsertBit(Bit bit)
{
if (_bits == null)
_bits = new Bit[ushort.MaxValue];
_bits[bit.OriginalNetworkId] = bit;
}
}
}

29
skyscraper8/Skyscraper/Text/AribStdB24TextDecoder.cs Normal file
View File

@ -0,0 +1,29 @@
namespace skyscraper8.Skyscraper.Text;
public class AribStdB24TextDecoder
{
private AribStdB24TextDecoder()
{
}
private static AribStdB24TextDecoder _instance;
public static AribStdB24TextDecoder GetInstance()
{
if (_instance == null)
{
_instance = new AribStdB24TextDecoder();
}
return _instance;
}
public string Decode(byte[] buffer)
{
if (buffer.Length == 0)
{
return "";
}
throw new NotImplementedException();
}
}