Refactored UNT handling.

skyscraper8/Dvb/SystemSoftwareUpdate/Model/UntJsonRepresentation.cs

@@ -0,0 +1,24 @@
+using skyscraper5.Dvb.SystemSoftwareUpdate.Model;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.Dvb.SystemSoftwareUpdate.Model
+{
+	public class UntJsonRepresentation : UpdateNotificationGroup
+	{
+		public UntJsonRepresentation() { }
+
+		public UntJsonRepresentation(UpdateNotificationGroup common, Compatibility compatibility, Platform platform)
+			: base(common)
+		{
+			Compatibility = compatibility;
+			Platform = platform;
+		}
+
+		public Compatibility Compatibility { get; }
+		public Platform Platform { get; }
+	}
+}

skyscraper8/Dvb/SystemSoftwareUpdate/Model/UpdateNotificationGroup.cs

@@ -1,4 +1,5 @@
-using System;
+using Newtonsoft.Json;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
@@ -11,7 +12,8 @@ namespace skyscraper5.Dvb.SystemSoftwareUpdate.Model
         public byte ActionType { get; }
         public byte OuiHash { get; }
         public string Oui { get; }
-        public byte ProcessingOrder { get; }
+		[JsonIgnore] public int OuiAsInt { get; private set; }
+		public byte ProcessingOrder { get; }
 
         //from descriptor 0x03 ssu location descriptor
         public ushort? DataBroadcastId { get; set; }
@@ -24,11 +26,31 @@ namespace skyscraper5.Dvb.SystemSoftwareUpdate.Model
         public int? UpdatePriority { get; set; }
 
 
-        public UpdateNotificationGroup(byte actionType, byte ouiHash, string oui, byte processingOrder)
+        protected UpdateNotificationGroup()
+        {
+
+        }
+
+        protected UpdateNotificationGroup(UpdateNotificationGroup copyFrom)
+        {
+            this.ActionType = copyFrom.ActionType;
+            this.OuiHash = copyFrom.OuiHash;
+            this.Oui = copyFrom.Oui;
+            this.ProcessingOrder = copyFrom.ProcessingOrder;
+            this.DataBroadcastId = copyFrom.DataBroadcastId;
+            this.AssociationTag = copyFrom.AssociationTag;
+            this.PrivateData = copyFrom.PrivateData;
+            this.UpdateFlag = copyFrom.UpdateFlag;
+            this.UpdateMethod = copyFrom.UpdateMethod;
+            this.UpdatePriority = copyFrom.UpdatePriority;
+        }
+
+        public UpdateNotificationGroup(byte actionType, byte ouiHash, byte[] oui, byte processingOrder)
         {
             ActionType = actionType;
             OuiHash = ouiHash;
-            Oui = oui;
+            Oui = Convert.ToHexString(oui);
+            OuiAsInt = oui[2] << 16 | oui[1] << 8 | oui[0];
             ProcessingOrder = processingOrder;
         }
 

skyscraper8/Dvb/SystemSoftwareUpdate/UntDecoder.cs

@@ -50,10 +50,10 @@ namespace skyscraper5.Dvb.SystemSoftwareUpdate
 
             byte sectionNumber = ms.ReadUInt8();
             byte lastSectionNumber = ms.ReadUInt8();
-            string oui = BitConverter.ToString(ms.ReadBytes(3));
+            byte[] ouiBytes = ms.ReadBytes(3);
             byte processingOrder = ms.ReadUInt8();
 
-            UpdateNotificationGroup common = new UpdateNotificationGroup(actionType, ouiHash, oui, processingOrder);
+            UpdateNotificationGroup common = new UpdateNotificationGroup(actionType, ouiHash, ouiBytes, processingOrder);
             CommonDescriptorLoop(ms, common);
             while (ms.GetAvailableBytes() > 4)
             {

skyscraper8/Mpeg2/Psi/Model/PmtStreamType.cs

@@ -50,5 +50,6 @@
         Iso23008_3Main = 0x2d,
         Iso23008_3Auxiliary = 0x2e,
         QualityAccessUnits = 0x2f,
+        Private193 = 0xc1
     }
 }

skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs

@@ -1,98 +1,106 @@
-using System;
+using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
-using System.Linq;
+using System.Linq;
-using System.Text;
+using System.Text;
-using System.Threading.Tasks;
+using System.Threading.Tasks;
-
+
-namespace skyscraper8.Skyscraper.Math
+namespace skyscraper8.Skyscraper.Math
-{
+{
-	public class EntropyCalculatorStream : Stream
+	public class EntropyCalculatorStream : Stream
-	{
+	{
-		public override void Flush()
+		public override void Flush()
-		{
+		{
-			//no actual writes done, so nothing to flush.
+			//no actual writes done, so nothing to flush.
-		}
+		}
-
+
-		public override int Read(byte[] buffer, int offset, int count)
+		public override int Read(byte[] buffer, int offset, int count)
-		{
+		{
-			//this is a sink, so nothing to read here.
+			//this is a sink, so nothing to read here.
-			throw new NotSupportedException();
+			throw new NotSupportedException();
-		}
+		}
-
+
-		public override long Seek(long offset, SeekOrigin origin)
+		public override long Seek(long offset, SeekOrigin origin)
-		{
+		{
-			if (offset == 0)
+			if (offset == 0)
-				return 0;
+				return 0;
-
+
-			//this is a sink, so we can't do actual seeks
+			//this is a sink, so we can't do actual seeks
-			throw new NotSupportedException();
+			throw new NotSupportedException();
-		}
+		}
-
+
-		public override void SetLength(long value)
+		public override void SetLength(long value)
-		{
+		{
-			//TODO: allow this later, although I have no idea why one would actually want this.
+			//TODO: allow this later, although I have no idea why one would actually want this.
-			throw new NotSupportedException();
+			throw new NotSupportedException();
-		}
+		}
-
+
-		private double _entropy;
+		private float _entropy;
-		private long[] freq;
+		private long[] freq;
-		private long internalPosition;
+		private long internalPosition;
-		public override void Write(byte[] buffer, int offset, int count)
+		private byte[] usedBytes;
-		{
+		public override void Write(byte[] buffer, int offset, int count)
-			//Count the bytes
+		{
-			if (freq == null)
+			
-				freq = new long[byte.MaxValue + 1];
+			//Count the bytes
-			for (int i = 0; i < count; i++)
+			if (freq == null)
-				freq[buffer[i + offset]]++;
+				freq = new long[byte.MaxValue + 1];
-			internalPosition += count;
+			for (int i = 0; i < count; i++)
-
+				freq[buffer[i + offset]]++;
-
+			internalPosition += count;
-			//Actually calculate the entropy here.
+
-			double newEntropy = 0.0;
+
-			for (int i = 0; i < freq.Length; i++)
+			//Actually calculate the entropy here.
-			{
+			float newEntropy = 0.0f;
-				double p_i = (double)freq[i] / (double)internalPosition;
+			for (int i = 0; i < freq.Length; i++)
-				if (p_i > 0)
+			{
-					newEntropy -= p_i * System.Math.Log2(p_i);
+				float p_i = (float)freq[i] / (float)internalPosition;
+				if (p_i > 0)
+				{
+					newEntropy -= p_i * FastMath.FastLog2(p_i);
+				}
 			}
-
+
-			_entropy = newEntropy;
+			_entropy = newEntropy;
-		}
+			if (_entropy > 8)
-
+			{
-		public override bool CanRead => false;
+				_entropy = 8;
-		public override bool CanSeek => false;
+			}
-		public override bool CanWrite => true;
+		}
-		public override long Length => internalPosition;
+
-		public override long Position
+		public override bool CanRead => false;
-		{
+		public override bool CanSeek => false;
-			get
+		public override bool CanWrite => true;
-			{
+		public override long Length => internalPosition;
-				return internalPosition;
+		public override long Position
-			}
+		{
-			set
+			get
-			{
+			{
-				throw new NotSupportedException();
+				return internalPosition;
-			}
+			}
-		}
+			set
-		public double Entropy
+			{
-		{
+				throw new NotSupportedException();
-			get
+			}
-			{
+		}
-				return _entropy;
+		public float Entropy
-			}
+		{
-		}
+			get
-
+			{
-		public double Percentage
+				return _entropy;
-		{
+			}
-			get
+		}
-			{
+
-				return (Entropy / 8.0) * 100.0;
+		public double Percentage
-			}
+		{
-		}
+			get
-
+			{
-		public void Cheat(double newValue)
+				return (Entropy / 8.0) * 100.0;
+			}
+		}
+
+		public void Cheat(float newValue)
 		{
 			_entropy = newValue;
-		}
+		}
-	}
+	}
-}
+}

skyscraper8/Skyscraper/Math/FastLog2.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.Skyscraper.Math
+{
+	static class FastMath
+	{
+		private const int LUT_BITS = 10; // 1024 entries
+		private const int LUT_SIZE = 1 << LUT_BITS;
+		private static readonly float[] log2Mantissa = new float[LUT_SIZE];
+
+		static FastMath()
+		{
+			for (int i = 0; i < LUT_SIZE; i++)
+			{
+				float m = 1.0f + (float)i / LUT_SIZE;
+				log2Mantissa[i] = MathF.Log2(m);
+			}
+		}
+
+		[MethodImpl(MethodImplOptions.AggressiveInlining)]
+		public static float FastLog2(float x)
+		{
+			int bits = Unsafe.As<float, int>(ref x);
+
+			int exponent = ((bits >> 23) & 0xFF) - 127;
+
+			// normalize mantissa
+			int mantissa = bits & 0x7FFFFF;
+			int idx = mantissa >> (23 - LUT_BITS);
+
+			return exponent + log2Mantissa[idx];
+		}
+	}
+}

skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs

@@ -1469,28 +1469,7 @@ namespace skyscraper5.Skyscraper.Scraper
         public void UpdateNotification(UpdateNotificationGroup common, UpdateNotificationTarget target, ushort ProgramNumber)
         {
             UiJunction?.OnSsuNotification(common, target, ProgramNumber);
-
+            DataStorage.StoreUpdateNotification2(common, target);
-            /*int hashCode = target.GetHashCode();
-            if (!ScraperStorage.TestForUpdateNotification(hashCode, common))
-            {
-                foreach (Compatibility compatibility in target.Compatibilities)
-                {
-                    foreach (Platform platform in target.Platforms)
-                    {
-                        ScraperStorage.StoreUpdateNotification(hashCode, common, compatibility, platform);
-                        LogEvent(SkyscraperContextEvent.UpdateNotification,
-                            String.Format("{0} -> {1}", compatibility, platform));
-                    }
-                }
-            }*/
-            foreach (Compatibility compatibility in target.Compatibilities)
-            {
-                foreach (Platform platform in target.Platforms)
-                {
-	                DataStorage.StoreUpdateNotification(0, common, compatibility, platform);
-                }
-            }
-
 
             if (common.AssociationTag.HasValue)
             {

skyscraper8/Skyscraper/Scraper/Storage/DataStorage.cs

@@ -63,7 +63,6 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
         void StoreAitApplication(AitApplication aitApplication);
         bool TestForCaSystem(int currentNetworkId, int currentTransportStreamId, int caDescriptorCaPid);
         void StoreCaSystem(int currentNetworkId, int currentTransportStreamId, CaDescriptor caDescriptor);
-        void StoreUpdateNotification(int hashCode, UpdateNotificationGroup common, Compatibility compatibility, Platform platform);
         bool TestForKnownRdsData(int currentNetworkId, int currentTransportStreamId, int programNumber);
         void EnableRdsCollection(int currentNetworkId, int currentTransportStreamId, int programNumber);
         bool UpdateRdsProgrammeServiceName(int currentNetworkId, int currentTransportStreamId, int programNumber, string programmeService2);
@@ -208,5 +207,6 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
 		void InsertRmt(Rmt rmt);
 		Tim GetLastTim(ushort interactiveNetworkId, PhysicalAddress physicalAddress);
 		void UpsertTim(ushort interactiveNetworkId, PhysicalAddress physicalAddress, Tim currentTim);
+		void StoreUpdateNotification2(UpdateNotificationGroup common, UpdateNotificationTarget target);
 	}
 }

skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs

@@ -386,35 +386,6 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
             File.WriteAllText(fi.FullName, JsonConvert.SerializeObject(caDescriptor, Formatting.Indented, jsonSerializerSettings));
         }
 
-        public bool TestForUpdateNotification(int hashCode, UpdateNotificationGroup common)
-        {
-            string combine = Path.Combine(rootDirectory.FullName, "UNT", common.Oui, "index.json");
-            return File.Exists(combine);
-        }
-
-        public void StoreUpdateNotification(int hashCode, UpdateNotificationGroup common, Compatibility compatibility, Platform platform)
-        { 
-            string combine = Path.Combine(rootDirectory.FullName, "UNT", common.Oui, compatibility.Version.ToString(), String.Format("{0}.json", platform.ToString()));
-            FileInfo platformFi = new FileInfo(combine);
-            EnsureDirectoryExists(platformFi.Directory);
-            if (!platformFi.Exists) 
-                File.WriteAllText(platformFi.FullName, JsonConvert.SerializeObject(platform, Formatting.Indented, jsonSerializerSettings));
-            else
-                return;
-
-            combine = Path.Combine(rootDirectory.FullName, "UNT", common.Oui, compatibility.Version.ToString(), "index.json");
-            FileInfo compatFi = new FileInfo(combine);
-            if (!compatFi.Exists)
-                File.WriteAllText(compatFi.FullName, JsonConvert.SerializeObject(compatibility, Formatting.Indented, jsonSerializerSettings));
-            else
-                return;
-
-            combine = Path.Combine(rootDirectory.FullName, "UNT", common.Oui, "index.json");
-            FileInfo commonFi = new FileInfo(combine);
-            if (!commonFi.Exists) 
-                File.WriteAllText(commonFi.FullName, JsonConvert.SerializeObject(common, Formatting.Indented, jsonSerializerSettings));
-        }
-
         public void DataCarouselModuleArrival(int currentNetworkId, int currentTransportStreamId, int elementaryPid, ushort moduleId, byte moduleVersion, Stream result)
         {
             string combine = Path.Combine(rootDirectory.FullName, "DSM-CC_Data", currentNetworkId.ToString(), currentTransportStreamId.ToString(), elementaryPid.ToString(), String.Format("{0}_V{1}.bin", moduleId, moduleVersion));
@@ -1927,5 +1898,10 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
 		{
 			throw new NotImplementedException();
 		}
+
+		public void StoreUpdateNotification2(UpdateNotificationGroup common, UpdateNotificationTarget target)
+		{
+			throw new NotImplementedException();
+		}
 	}
 }

skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs

@@ -1927,5 +1927,10 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.InMemory
 			Tuple<ushort, PhysicalAddress> key = new Tuple<ushort, PhysicalAddress>(interactiveNetworkId, physicalAddress);
 			tims[key] = currentTim;
 		}
+
+		public void StoreUpdateNotification2(UpdateNotificationGroup common, UpdateNotificationTarget target)
+		{
+			throw new NotImplementedException();
+		}
 	}
 }