From f605eb33c8343b905f429c66bb7d0c26c5263342 Mon Sep 17 00:00:00 2001 From: feyris-tan <4116042+feyris-tan@users.noreply.github.com> Date: Mon, 23 Mar 2026 23:37:23 +0100 Subject: [PATCH] Refactored UNT handling. --- .../Model/UntJsonRepresentation.cs | 24 +++ .../Model/UpdateNotificationGroup.cs | 30 ++- .../Dvb/SystemSoftwareUpdate/UntDecoder.cs | 4 +- skyscraper8/Mpeg2/Psi/Model/PmtStreamType.cs | 1 + .../Math/EntropyCalculatorStream.cs | 198 +++++++++--------- skyscraper8/Skyscraper/Math/FastLog2.cs | 39 ++++ .../Skyscraper/Scraper/SkyscraperContext.cs | 23 +- .../Skyscraper/Scraper/Storage/DataStorage.cs | 2 +- .../Storage/Filesystem/FilesystemStorage.cs | 34 +-- .../InMemory/InMemoryScraperStorage.cs | 5 + 10 files changed, 207 insertions(+), 153 deletions(-) create mode 100644 skyscraper8/Dvb/SystemSoftwareUpdate/Model/UntJsonRepresentation.cs create mode 100644 skyscraper8/Skyscraper/Math/FastLog2.cs diff --git a/skyscraper8/Dvb/SystemSoftwareUpdate/Model/UntJsonRepresentation.cs b/skyscraper8/Dvb/SystemSoftwareUpdate/Model/UntJsonRepresentation.cs new file mode 100644 index 0000000..b640032 --- /dev/null +++ b/skyscraper8/Dvb/SystemSoftwareUpdate/Model/UntJsonRepresentation.cs @@ -0,0 +1,24 @@ +using skyscraper5.Dvb.SystemSoftwareUpdate.Model; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.Dvb.SystemSoftwareUpdate.Model +{ + public class UntJsonRepresentation : UpdateNotificationGroup + { + public UntJsonRepresentation() { } + + public UntJsonRepresentation(UpdateNotificationGroup common, Compatibility compatibility, Platform platform) + : base(common) + { + Compatibility = compatibility; + Platform = platform; + } + + public Compatibility Compatibility { get; } + public Platform Platform { get; } + } +} diff --git a/skyscraper8/Dvb/SystemSoftwareUpdate/Model/UpdateNotificationGroup.cs b/skyscraper8/Dvb/SystemSoftwareUpdate/Model/UpdateNotificationGroup.cs index b40fc5e..362fa7f 100644 --- a/skyscraper8/Dvb/SystemSoftwareUpdate/Model/UpdateNotificationGroup.cs +++ b/skyscraper8/Dvb/SystemSoftwareUpdate/Model/UpdateNotificationGroup.cs @@ -1,4 +1,5 @@ -using System; +using Newtonsoft.Json; +using System; using System.Collections.Generic; using System.Linq; using System.Text; @@ -11,7 +12,8 @@ namespace skyscraper5.Dvb.SystemSoftwareUpdate.Model public byte ActionType { get; } public byte OuiHash { get; } public string Oui { get; } - public byte ProcessingOrder { get; } + [JsonIgnore] public int OuiAsInt { get; private set; } + public byte ProcessingOrder { get; } //from descriptor 0x03 ssu location descriptor public ushort? DataBroadcastId { get; set; } @@ -24,11 +26,31 @@ namespace skyscraper5.Dvb.SystemSoftwareUpdate.Model public int? UpdatePriority { get; set; } - public UpdateNotificationGroup(byte actionType, byte ouiHash, string oui, byte processingOrder) + protected UpdateNotificationGroup() + { + + } + + protected UpdateNotificationGroup(UpdateNotificationGroup copyFrom) + { + this.ActionType = copyFrom.ActionType; + this.OuiHash = copyFrom.OuiHash; + this.Oui = copyFrom.Oui; + this.ProcessingOrder = copyFrom.ProcessingOrder; + this.DataBroadcastId = copyFrom.DataBroadcastId; + this.AssociationTag = copyFrom.AssociationTag; + this.PrivateData = copyFrom.PrivateData; + this.UpdateFlag = copyFrom.UpdateFlag; + this.UpdateMethod = copyFrom.UpdateMethod; + this.UpdatePriority = copyFrom.UpdatePriority; + } + + public UpdateNotificationGroup(byte actionType, byte ouiHash, byte[] oui, byte processingOrder) { ActionType = actionType; OuiHash = ouiHash; - Oui = oui; + Oui = Convert.ToHexString(oui); + OuiAsInt = oui[2] << 16 | oui[1] << 8 | oui[0]; ProcessingOrder = processingOrder; } diff --git a/skyscraper8/Dvb/SystemSoftwareUpdate/UntDecoder.cs b/skyscraper8/Dvb/SystemSoftwareUpdate/UntDecoder.cs index 4110d43..ee003b2 100644 --- a/skyscraper8/Dvb/SystemSoftwareUpdate/UntDecoder.cs +++ b/skyscraper8/Dvb/SystemSoftwareUpdate/UntDecoder.cs @@ -50,10 +50,10 @@ namespace skyscraper5.Dvb.SystemSoftwareUpdate byte sectionNumber = ms.ReadUInt8(); byte lastSectionNumber = ms.ReadUInt8(); - string oui = BitConverter.ToString(ms.ReadBytes(3)); + byte[] ouiBytes = ms.ReadBytes(3); byte processingOrder = ms.ReadUInt8(); - UpdateNotificationGroup common = new UpdateNotificationGroup(actionType, ouiHash, oui, processingOrder); + UpdateNotificationGroup common = new UpdateNotificationGroup(actionType, ouiHash, ouiBytes, processingOrder); CommonDescriptorLoop(ms, common); while (ms.GetAvailableBytes() > 4) { diff --git a/skyscraper8/Mpeg2/Psi/Model/PmtStreamType.cs b/skyscraper8/Mpeg2/Psi/Model/PmtStreamType.cs index 6b14090..859f0ba 100644 --- a/skyscraper8/Mpeg2/Psi/Model/PmtStreamType.cs +++ b/skyscraper8/Mpeg2/Psi/Model/PmtStreamType.cs @@ -50,5 +50,6 @@ Iso23008_3Main = 0x2d, Iso23008_3Auxiliary = 0x2e, QualityAccessUnits = 0x2f, + Private193 = 0xc1 } } diff --git a/skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs b/skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs index 69b38e0..a1a8044 100644 --- a/skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs +++ b/skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs @@ -1,98 +1,106 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace skyscraper8.Skyscraper.Math -{ - public class EntropyCalculatorStream : Stream - { - public override void Flush() - { - //no actual writes done, so nothing to flush. - } - - public override int Read(byte[] buffer, int offset, int count) - { - //this is a sink, so nothing to read here. - throw new NotSupportedException(); - } - - public override long Seek(long offset, SeekOrigin origin) - { - if (offset == 0) - return 0; - - //this is a sink, so we can't do actual seeks - throw new NotSupportedException(); - } - - public override void SetLength(long value) - { - //TODO: allow this later, although I have no idea why one would actually want this. - throw new NotSupportedException(); - } - - private double _entropy; - private long[] freq; - private long internalPosition; - public override void Write(byte[] buffer, int offset, int count) - { - //Count the bytes - if (freq == null) - freq = new long[byte.MaxValue + 1]; - for (int i = 0; i < count; i++) - freq[buffer[i + offset]]++; - internalPosition += count; - - - //Actually calculate the entropy here. - double newEntropy = 0.0; - for (int i = 0; i < freq.Length; i++) - { - double p_i = (double)freq[i] / (double)internalPosition; - if (p_i > 0) - newEntropy -= p_i * System.Math.Log2(p_i); +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.Skyscraper.Math +{ + public class EntropyCalculatorStream : Stream + { + public override void Flush() + { + //no actual writes done, so nothing to flush. + } + + public override int Read(byte[] buffer, int offset, int count) + { + //this is a sink, so nothing to read here. + throw new NotSupportedException(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + if (offset == 0) + return 0; + + //this is a sink, so we can't do actual seeks + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + //TODO: allow this later, although I have no idea why one would actually want this. + throw new NotSupportedException(); + } + + private float _entropy; + private long[] freq; + private long internalPosition; + private byte[] usedBytes; + public override void Write(byte[] buffer, int offset, int count) + { + + //Count the bytes + if (freq == null) + freq = new long[byte.MaxValue + 1]; + for (int i = 0; i < count; i++) + freq[buffer[i + offset]]++; + internalPosition += count; + + + //Actually calculate the entropy here. + float newEntropy = 0.0f; + for (int i = 0; i < freq.Length; i++) + { + float p_i = (float)freq[i] / (float)internalPosition; + if (p_i > 0) + { + newEntropy -= p_i * FastMath.FastLog2(p_i); + } } - - _entropy = newEntropy; - } - - public override bool CanRead => false; - public override bool CanSeek => false; - public override bool CanWrite => true; - public override long Length => internalPosition; - public override long Position - { - get - { - return internalPosition; - } - set - { - throw new NotSupportedException(); - } - } - public double Entropy - { - get - { - return _entropy; - } - } - - public double Percentage - { - get - { - return (Entropy / 8.0) * 100.0; - } - } - - public void Cheat(double newValue) + + _entropy = newEntropy; + if (_entropy > 8) + { + _entropy = 8; + } + } + + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => true; + public override long Length => internalPosition; + public override long Position + { + get + { + return internalPosition; + } + set + { + throw new NotSupportedException(); + } + } + public float Entropy + { + get + { + return _entropy; + } + } + + public double Percentage + { + get + { + return (Entropy / 8.0) * 100.0; + } + } + + public void Cheat(float newValue) { _entropy = newValue; - } - } -} + } + } +} diff --git a/skyscraper8/Skyscraper/Math/FastLog2.cs b/skyscraper8/Skyscraper/Math/FastLog2.cs new file mode 100644 index 0000000..04c3ada --- /dev/null +++ b/skyscraper8/Skyscraper/Math/FastLog2.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading.Tasks; + +namespace skyscraper8.Skyscraper.Math +{ + static class FastMath + { + private const int LUT_BITS = 10; // 1024 entries + private const int LUT_SIZE = 1 << LUT_BITS; + private static readonly float[] log2Mantissa = new float[LUT_SIZE]; + + static FastMath() + { + for (int i = 0; i < LUT_SIZE; i++) + { + float m = 1.0f + (float)i / LUT_SIZE; + log2Mantissa[i] = MathF.Log2(m); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static float FastLog2(float x) + { + int bits = Unsafe.As(ref x); + + int exponent = ((bits >> 23) & 0xFF) - 127; + + // normalize mantissa + int mantissa = bits & 0x7FFFFF; + int idx = mantissa >> (23 - LUT_BITS); + + return exponent + log2Mantissa[idx]; + } + } +} diff --git a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs index 2a6a716..85645f0 100644 --- a/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs +++ b/skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs @@ -1469,28 +1469,7 @@ namespace skyscraper5.Skyscraper.Scraper public void UpdateNotification(UpdateNotificationGroup common, UpdateNotificationTarget target, ushort ProgramNumber) { UiJunction?.OnSsuNotification(common, target, ProgramNumber); - - /*int hashCode = target.GetHashCode(); - if (!ScraperStorage.TestForUpdateNotification(hashCode, common)) - { - foreach (Compatibility compatibility in target.Compatibilities) - { - foreach (Platform platform in target.Platforms) - { - ScraperStorage.StoreUpdateNotification(hashCode, common, compatibility, platform); - LogEvent(SkyscraperContextEvent.UpdateNotification, - String.Format("{0} -> {1}", compatibility, platform)); - } - } - }*/ - foreach (Compatibility compatibility in target.Compatibilities) - { - foreach (Platform platform in target.Platforms) - { - DataStorage.StoreUpdateNotification(0, common, compatibility, platform); - } - } - + DataStorage.StoreUpdateNotification2(common, target); if (common.AssociationTag.HasValue) { diff --git a/skyscraper8/Skyscraper/Scraper/Storage/DataStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/DataStorage.cs index ecf7dfa..b351f78 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/DataStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/DataStorage.cs @@ -63,7 +63,6 @@ namespace skyscraper8.Skyscraper.Scraper.Storage void StoreAitApplication(AitApplication aitApplication); bool TestForCaSystem(int currentNetworkId, int currentTransportStreamId, int caDescriptorCaPid); void StoreCaSystem(int currentNetworkId, int currentTransportStreamId, CaDescriptor caDescriptor); - void StoreUpdateNotification(int hashCode, UpdateNotificationGroup common, Compatibility compatibility, Platform platform); bool TestForKnownRdsData(int currentNetworkId, int currentTransportStreamId, int programNumber); void EnableRdsCollection(int currentNetworkId, int currentTransportStreamId, int programNumber); bool UpdateRdsProgrammeServiceName(int currentNetworkId, int currentTransportStreamId, int programNumber, string programmeService2); @@ -208,5 +207,6 @@ namespace skyscraper8.Skyscraper.Scraper.Storage void InsertRmt(Rmt rmt); Tim GetLastTim(ushort interactiveNetworkId, PhysicalAddress physicalAddress); void UpsertTim(ushort interactiveNetworkId, PhysicalAddress physicalAddress, Tim currentTim); + void StoreUpdateNotification2(UpdateNotificationGroup common, UpdateNotificationTarget target); } } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs index fe1e1b0..bc33494 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs @@ -386,35 +386,6 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem File.WriteAllText(fi.FullName, JsonConvert.SerializeObject(caDescriptor, Formatting.Indented, jsonSerializerSettings)); } - public bool TestForUpdateNotification(int hashCode, UpdateNotificationGroup common) - { - string combine = Path.Combine(rootDirectory.FullName, "UNT", common.Oui, "index.json"); - return File.Exists(combine); - } - - public void StoreUpdateNotification(int hashCode, UpdateNotificationGroup common, Compatibility compatibility, Platform platform) - { - string combine = Path.Combine(rootDirectory.FullName, "UNT", common.Oui, compatibility.Version.ToString(), String.Format("{0}.json", platform.ToString())); - FileInfo platformFi = new FileInfo(combine); - EnsureDirectoryExists(platformFi.Directory); - if (!platformFi.Exists) - File.WriteAllText(platformFi.FullName, JsonConvert.SerializeObject(platform, Formatting.Indented, jsonSerializerSettings)); - else - return; - - combine = Path.Combine(rootDirectory.FullName, "UNT", common.Oui, compatibility.Version.ToString(), "index.json"); - FileInfo compatFi = new FileInfo(combine); - if (!compatFi.Exists) - File.WriteAllText(compatFi.FullName, JsonConvert.SerializeObject(compatibility, Formatting.Indented, jsonSerializerSettings)); - else - return; - - combine = Path.Combine(rootDirectory.FullName, "UNT", common.Oui, "index.json"); - FileInfo commonFi = new FileInfo(combine); - if (!commonFi.Exists) - File.WriteAllText(commonFi.FullName, JsonConvert.SerializeObject(common, Formatting.Indented, jsonSerializerSettings)); - } - public void DataCarouselModuleArrival(int currentNetworkId, int currentTransportStreamId, int elementaryPid, ushort moduleId, byte moduleVersion, Stream result) { string combine = Path.Combine(rootDirectory.FullName, "DSM-CC_Data", currentNetworkId.ToString(), currentTransportStreamId.ToString(), elementaryPid.ToString(), String.Format("{0}_V{1}.bin", moduleId, moduleVersion)); @@ -1927,5 +1898,10 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem { throw new NotImplementedException(); } + + public void StoreUpdateNotification2(UpdateNotificationGroup common, UpdateNotificationTarget target) + { + throw new NotImplementedException(); + } } } diff --git a/skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs b/skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs index 2c1bd55..a65711c 100644 --- a/skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs +++ b/skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs @@ -1927,5 +1927,10 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.InMemory Tuple key = new Tuple(interactiveNetworkId, physicalAddress); tims[key] = currentTim; } + + public void StoreUpdateNotification2(UpdateNotificationGroup common, UpdateNotificationTarget target) + { + throw new NotImplementedException(); + } } }