Refactored UNT handling.

skyscraper8/Dvb/SystemSoftwareUpdate/Model/UntJsonRepresentation.cs

@@ -0,0 +1,24 @@
+using skyscraper5.Dvb.SystemSoftwareUpdate.Model;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.Dvb.SystemSoftwareUpdate.Model
+{
+	public class UntJsonRepresentation : UpdateNotificationGroup
+	{
+		public UntJsonRepresentation() { }
+
+		public UntJsonRepresentation(UpdateNotificationGroup common, Compatibility compatibility, Platform platform)
+			: base(common)
+		{
+			Compatibility = compatibility;
+			Platform = platform;
+		}
+
+		public Compatibility Compatibility { get; }
+		public Platform Platform { get; }
+	}
+}

skyscraper8/Dvb/SystemSoftwareUpdate/Model/UpdateNotificationGroup.cs

@@ -1,4 +1,5 @@
-using System;
+using Newtonsoft.Json;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
@@ -11,6 +12,7 @@ namespace skyscraper5.Dvb.SystemSoftwareUpdate.Model
         public byte ActionType { get; }
         public byte OuiHash { get; }
         public string Oui { get; }
+		[JsonIgnore] public int OuiAsInt { get; private set; }
 		public byte ProcessingOrder { get; }
 
         //from descriptor 0x03 ssu location descriptor
@@ -24,11 +26,31 @@ namespace skyscraper5.Dvb.SystemSoftwareUpdate.Model
         public int? UpdatePriority { get; set; }
 
 
-        public UpdateNotificationGroup(byte actionType, byte ouiHash, string oui, byte processingOrder)
+        protected UpdateNotificationGroup()
+        {
+
+        }
+
+        protected UpdateNotificationGroup(UpdateNotificationGroup copyFrom)
+        {
+            this.ActionType = copyFrom.ActionType;
+            this.OuiHash = copyFrom.OuiHash;
+            this.Oui = copyFrom.Oui;
+            this.ProcessingOrder = copyFrom.ProcessingOrder;
+            this.DataBroadcastId = copyFrom.DataBroadcastId;
+            this.AssociationTag = copyFrom.AssociationTag;
+            this.PrivateData = copyFrom.PrivateData;
+            this.UpdateFlag = copyFrom.UpdateFlag;
+            this.UpdateMethod = copyFrom.UpdateMethod;
+            this.UpdatePriority = copyFrom.UpdatePriority;
+        }
+
+        public UpdateNotificationGroup(byte actionType, byte ouiHash, byte[] oui, byte processingOrder)
         {
             ActionType = actionType;
             OuiHash = ouiHash;
-            Oui = oui;
+            Oui = Convert.ToHexString(oui);
+            OuiAsInt = oui[2] << 16 | oui[1] << 8 | oui[0];
             ProcessingOrder = processingOrder;
         }
 

skyscraper8/Dvb/SystemSoftwareUpdate/UntDecoder.cs

@@ -50,10 +50,10 @@ namespace skyscraper5.Dvb.SystemSoftwareUpdate
 
             byte sectionNumber = ms.ReadUInt8();
             byte lastSectionNumber = ms.ReadUInt8();
-            string oui = BitConverter.ToString(ms.ReadBytes(3));
+            byte[] ouiBytes = ms.ReadBytes(3);
             byte processingOrder = ms.ReadUInt8();
 
-            UpdateNotificationGroup common = new UpdateNotificationGroup(actionType, ouiHash, oui, processingOrder);
+            UpdateNotificationGroup common = new UpdateNotificationGroup(actionType, ouiHash, ouiBytes, processingOrder);
             CommonDescriptorLoop(ms, common);
             while (ms.GetAvailableBytes() > 4)
             {

skyscraper8/Mpeg2/Psi/Model/PmtStreamType.cs

@@ -50,5 +50,6 @@
         Iso23008_3Main = 0x2d,
         Iso23008_3Auxiliary = 0x2e,
         QualityAccessUnits = 0x2f,
+        Private193 = 0xc1
     }
 }

skyscraper8/Skyscraper/Math/EntropyCalculatorStream.cs

@@ -34,11 +34,13 @@ namespace skyscraper8.Skyscraper.Math
 			throw new NotSupportedException();
 		}
 
-		private double _entropy;
+		private float _entropy;
 		private long[] freq;
 		private long internalPosition;
+		private byte[] usedBytes;
 		public override void Write(byte[] buffer, int offset, int count)
 		{
+			
 			//Count the bytes
 			if (freq == null)
 				freq = new long[byte.MaxValue + 1];
@@ -48,15 +50,21 @@ namespace skyscraper8.Skyscraper.Math
 
 
 			//Actually calculate the entropy here.
-			double newEntropy = 0.0;
+			float newEntropy = 0.0f;
 			for (int i = 0; i < freq.Length; i++)
 			{
-				double p_i = (double)freq[i] / (double)internalPosition;
+				float p_i = (float)freq[i] / (float)internalPosition;
 				if (p_i > 0)
-					newEntropy -= p_i * System.Math.Log2(p_i);
+				{
+					newEntropy -= p_i * FastMath.FastLog2(p_i);
+				}
 			}
 
 			_entropy = newEntropy;
+			if (_entropy > 8)
+			{
+				_entropy = 8;
+			}
 		}
 
 		public override bool CanRead => false;
@@ -74,7 +82,7 @@ namespace skyscraper8.Skyscraper.Math
 				throw new NotSupportedException();
 			}
 		}
-		public double Entropy
+		public float Entropy
 		{
 			get
 			{
@@ -90,7 +98,7 @@ namespace skyscraper8.Skyscraper.Math
 			}
 		}
 
-		public void Cheat(double newValue)
+		public void Cheat(float newValue)
 		{
 			_entropy = newValue;
 		}

skyscraper8/Skyscraper/Math/FastLog2.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace skyscraper8.Skyscraper.Math
+{
+	static class FastMath
+	{
+		private const int LUT_BITS = 10; // 1024 entries
+		private const int LUT_SIZE = 1 << LUT_BITS;
+		private static readonly float[] log2Mantissa = new float[LUT_SIZE];
+
+		static FastMath()
+		{
+			for (int i = 0; i < LUT_SIZE; i++)
+			{
+				float m = 1.0f + (float)i / LUT_SIZE;
+				log2Mantissa[i] = MathF.Log2(m);
+			}
+		}
+
+		[MethodImpl(MethodImplOptions.AggressiveInlining)]
+		public static float FastLog2(float x)
+		{
+			int bits = Unsafe.As<float, int>(ref x);
+
+			int exponent = ((bits >> 23) & 0xFF) - 127;
+
+			// normalize mantissa
+			int mantissa = bits & 0x7FFFFF;
+			int idx = mantissa >> (23 - LUT_BITS);
+
+			return exponent + log2Mantissa[idx];
+		}
+	}
+}

skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs

@@ -1469,28 +1469,7 @@ namespace skyscraper5.Skyscraper.Scraper
         public void UpdateNotification(UpdateNotificationGroup common, UpdateNotificationTarget target, ushort ProgramNumber)
         {
             UiJunction?.OnSsuNotification(common, target, ProgramNumber);
-
+            DataStorage.StoreUpdateNotification2(common, target);
-            /*int hashCode = target.GetHashCode();
-            if (!ScraperStorage.TestForUpdateNotification(hashCode, common))
-            {
-                foreach (Compatibility compatibility in target.Compatibilities)
-                {
-                    foreach (Platform platform in target.Platforms)
-                    {
-                        ScraperStorage.StoreUpdateNotification(hashCode, common, compatibility, platform);
-                        LogEvent(SkyscraperContextEvent.UpdateNotification,
-                            String.Format("{0} -> {1}", compatibility, platform));
-                    }
-                }
-            }*/
-            foreach (Compatibility compatibility in target.Compatibilities)
-            {
-                foreach (Platform platform in target.Platforms)
-                {
-	                DataStorage.StoreUpdateNotification(0, common, compatibility, platform);
-                }
-            }
-
 
             if (common.AssociationTag.HasValue)
             {

skyscraper8/Skyscraper/Scraper/Storage/DataStorage.cs

@@ -63,7 +63,6 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
         void StoreAitApplication(AitApplication aitApplication);
         bool TestForCaSystem(int currentNetworkId, int currentTransportStreamId, int caDescriptorCaPid);
         void StoreCaSystem(int currentNetworkId, int currentTransportStreamId, CaDescriptor caDescriptor);
-        void StoreUpdateNotification(int hashCode, UpdateNotificationGroup common, Compatibility compatibility, Platform platform);
         bool TestForKnownRdsData(int currentNetworkId, int currentTransportStreamId, int programNumber);
         void EnableRdsCollection(int currentNetworkId, int currentTransportStreamId, int programNumber);
         bool UpdateRdsProgrammeServiceName(int currentNetworkId, int currentTransportStreamId, int programNumber, string programmeService2);
@@ -208,5 +207,6 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
 		void InsertRmt(Rmt rmt);
 		Tim GetLastTim(ushort interactiveNetworkId, PhysicalAddress physicalAddress);
 		void UpsertTim(ushort interactiveNetworkId, PhysicalAddress physicalAddress, Tim currentTim);
+		void StoreUpdateNotification2(UpdateNotificationGroup common, UpdateNotificationTarget target);
 	}
 }

skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs

@@ -386,35 +386,6 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
             File.WriteAllText(fi.FullName, JsonConvert.SerializeObject(caDescriptor, Formatting.Indented, jsonSerializerSettings));
         }
 
-        public bool TestForUpdateNotification(int hashCode, UpdateNotificationGroup common)
-        {
-            string combine = Path.Combine(rootDirectory.FullName, "UNT", common.Oui, "index.json");
-            return File.Exists(combine);
-        }
-
-        public void StoreUpdateNotification(int hashCode, UpdateNotificationGroup common, Compatibility compatibility, Platform platform)
-        { 
-            string combine = Path.Combine(rootDirectory.FullName, "UNT", common.Oui, compatibility.Version.ToString(), String.Format("{0}.json", platform.ToString()));
-            FileInfo platformFi = new FileInfo(combine);
-            EnsureDirectoryExists(platformFi.Directory);
-            if (!platformFi.Exists) 
-                File.WriteAllText(platformFi.FullName, JsonConvert.SerializeObject(platform, Formatting.Indented, jsonSerializerSettings));
-            else
-                return;
-
-            combine = Path.Combine(rootDirectory.FullName, "UNT", common.Oui, compatibility.Version.ToString(), "index.json");
-            FileInfo compatFi = new FileInfo(combine);
-            if (!compatFi.Exists)
-                File.WriteAllText(compatFi.FullName, JsonConvert.SerializeObject(compatibility, Formatting.Indented, jsonSerializerSettings));
-            else
-                return;
-
-            combine = Path.Combine(rootDirectory.FullName, "UNT", common.Oui, "index.json");
-            FileInfo commonFi = new FileInfo(combine);
-            if (!commonFi.Exists) 
-                File.WriteAllText(commonFi.FullName, JsonConvert.SerializeObject(common, Formatting.Indented, jsonSerializerSettings));
-        }
-
         public void DataCarouselModuleArrival(int currentNetworkId, int currentTransportStreamId, int elementaryPid, ushort moduleId, byte moduleVersion, Stream result)
         {
             string combine = Path.Combine(rootDirectory.FullName, "DSM-CC_Data", currentNetworkId.ToString(), currentTransportStreamId.ToString(), elementaryPid.ToString(), String.Format("{0}_V{1}.bin", moduleId, moduleVersion));
@@ -1927,5 +1898,10 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
 		{
 			throw new NotImplementedException();
 		}
+
+		public void StoreUpdateNotification2(UpdateNotificationGroup common, UpdateNotificationTarget target)
+		{
+			throw new NotImplementedException();
+		}
 	}
 }

skyscraper8/Skyscraper/Scraper/Storage/InMemory/InMemoryScraperStorage.cs

@@ -1927,5 +1927,10 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.InMemory
 			Tuple<ushort, PhysicalAddress> key = new Tuple<ushort, PhysicalAddress>(interactiveNetworkId, physicalAddress);
 			tims[key] = currentTim;
 		}
+
+		public void StoreUpdateNotification2(UpdateNotificationGroup common, UpdateNotificationTarget target)
+		{
+			throw new NotImplementedException();
+		}
 	}
 }