using CommunityToolkit.HighPerformance.Buffers; using Minio; using Minio.DataModel; using Minio.DataModel.Args; using Minio.DataModel.Response; using Minio.Exceptions; using skyscraper5.Dvb.DataBroadcasting.SkyscraperVfs; using skyscraper5.Dvb.Descriptors; using skyscraper8.DvbNip; using skyscraper8.Experimentals.NdsSsu; using skyscraper8.Ietf.FLUTE; using skyscraper8.Skyscraper.Drawing; using skyscraper8.Skyscraper.Scraper.Storage; using System; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.IO; using System.Net; using System.Runtime.InteropServices; using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; using skyscraper5.Skyscraper; using skyscraper8.SimpleServiceDiscoveryProtocol; namespace skyscraper5.Data { public class MinioObjectStorage : ObjectStorage { private readonly IMinioClient _minioClient; private readonly string _minioBucket; private List _tasks; private List droppedFiles; public MinioObjectStorage(IMinioClient minioClient, string minioBucket) { _minioClient = minioClient; _minioBucket = minioBucket; _tasks = new List(); droppedFiles = new List(); } private void WriteObject(string fullName, Stream buffer, string mime = "application/octet-stream", Dictionary optionalData = null) { PutObjectArgs putObjectArgs = new PutObjectArgs(); putObjectArgs = putObjectArgs.WithBucket(_minioBucket); putObjectArgs = putObjectArgs.WithObject(fullName); putObjectArgs = putObjectArgs.WithStreamData(buffer); //putObjectArgs = putObjectArgs.WithObjectSize(buffer.Length); putObjectArgs = putObjectArgs.WithContentType(mime); putObjectArgs = putObjectArgs.WithHeaders(optionalData); if (buffer.CanSeek) { putObjectArgs = putObjectArgs.WithObjectSize(buffer.Length); } else if (optionalData.ContainsKey("X-Skyscraper-Content-Length")) { putObjectArgs = putObjectArgs.WithObjectSize(long.Parse(optionalData["X-Skyscraper-Content-Length"])); } if (definetlyKnownFiles == null) definetlyKnownFiles = new HashSet(); if (definetlyMissingFiles == null) definetlyMissingFiles = new HashSet(); lock (_tasks) { _tasks.Add(_minioClient.PutObjectAsync(putObjectArgs).ContinueWith(task => { if (task.IsFaulted) { throw new MinioException("A minio upload task failed."); } droppedFiles.Add(fullName); Monitor.Enter(definetlyKnownFiles); definetlyKnownFiles.Add(fullName); Monitor.Exit(definetlyKnownFiles); Monitor.Enter(definetlyMissingFiles); definetlyMissingFiles.Remove(fullName); Monitor.Exit(definetlyMissingFiles); buffer.Close(); buffer.Dispose(); })); } CleanTaskList(); } private byte[] GetObject(string fullName) { StatObjectArgs statObjectArgs = new StatObjectArgs(); statObjectArgs.WithObject(fullName); statObjectArgs.WithBucket(_minioBucket); Task async = _minioClient.StatObjectAsync(statObjectArgs); ObjectStat asyncResult = async.Result; ObjectRetriever objectRetriever = new ObjectRetriever(asyncResult.Size); GetObjectArgs getObjectArgs = new GetObjectArgs(); getObjectArgs.WithBucket(_minioBucket); getObjectArgs = getObjectArgs.WithObject(fullName); getObjectArgs.WithCallbackStream(objectRetriever.OurAction); Task objectAsync = _minioClient.GetObjectAsync(getObjectArgs); ObjectStat objectStat = objectAsync.Result; return objectRetriever.GetBuffer(); } public class ObjectRetriever { public ObjectRetriever(long size) { buffer = new byte[size]; } private byte[] buffer; public void OurAction(Stream obj) { int read = obj.Read(buffer, 0, (int)buffer.Length); if (read != buffer.Length) throw new MinioException("incomplete read"); } public byte[] GetBuffer() { return buffer; } } private void CleanTaskList() { while (true) { bool done = true; lock (_tasks) { foreach (Task task in _tasks) { if (task.IsCompleted) { _tasks.Remove(task); Debug.WriteLine(String.Format("Removed completed Task: {0}", task.ToString())); done = false; break; } } } if (done) break; } } public bool ObjectCarouselFileArrival(VfsFile vfsFile, int transportStreamId, int networkId) { string combine = string.Join('/', "DSM-CC_Objects", networkId.ToString(), transportStreamId.ToString(), vfsFile.SourcePid.ToString(), vfsFile.ToString().Substring(1)); combine = combine.Replace('\\', '/'); if (FileExists(combine)) return false; Dictionary extendedInfo = new Dictionary(); extendedInfo.Add(nameof(vfsFile.ObjectLocation.CarouselId), vfsFile.ObjectLocation.CarouselId.ToString()); extendedInfo.Add(nameof(vfsFile.ObjectLocation.ModuleId), vfsFile.ObjectLocation.ModuleId.ToString()); extendedInfo.Add(nameof(vfsFile.ObjectLocation.Version), vfsFile.ObjectLocation.Version.ToString()); extendedInfo.Add(nameof(vfsFile.ObjectLocation.ObjectKey), BitConverter.ToString(vfsFile.ObjectLocation.ObjectKey)); WriteObject(combine,new MemoryStream(vfsFile.FileContent),"application/octet-stream",extendedInfo); return true; } public void DataCarouselModuleArrival(int currentNetworkId, int currentTransportStreamId, int elementaryPid, ushort moduleModuleId, byte moduleModuleVersion, Stream result) { string combine = string.Join('/', "DSM-CC_Data", currentNetworkId.ToString(), currentTransportStreamId.ToString(), elementaryPid.ToString(), String.Format("{0}_V{1}.bin", moduleModuleId, moduleModuleVersion)); if (FileExists(combine)) return; WriteObject(combine, result); WantedModuleCoordinate coordinate = new WantedModuleCoordinate(currentNetworkId, currentTransportStreamId, elementaryPid, moduleModuleId, moduleModuleVersion); knownModuleCoordinates.Add(coordinate); wantedModuleCoordinates.Remove(coordinate); } public bool FileExists(string combine) { if (definetlyMissingFiles == null) definetlyMissingFiles = new HashSet(); if (definetlyKnownFiles == null) definetlyKnownFiles = new HashSet(); if (definetlyKnownFiles.Contains(combine)) return true; if (definetlyMissingFiles.Contains(combine)) return false; if (droppedFiles != null) { if (droppedFiles.Contains(combine)) return true; } StatObjectArgs soa = new StatObjectArgs().WithBucket(_minioBucket).WithObject(combine); try { ObjectStat objectStat = _minioClient.StatObjectAsync(soa).Result; definetlyKnownFiles.Add(combine); return true; } catch (AggregateException e) { MinioException minioException = e.InnerExceptions[0] as MinioException; ObjectNotFoundException objectNotFoundException = minioException as ObjectNotFoundException; if (objectNotFoundException != null) { Monitor.Enter(definetlyMissingFiles); definetlyMissingFiles.Add(combine); Monitor.Exit(definetlyMissingFiles); return false; } else { throw minioException; } } } public bool IsDsmCcModuleWanted(int currentNetworkId, int currentTransportStreamId, int elementaryPid, ushort moduleId, byte moduleVersion) { if (wantedModuleCoordinates == null) wantedModuleCoordinates = new HashSet(); if (knownModuleCoordinates == null) knownModuleCoordinates = new HashSet(); WantedModuleCoordinate coordinate = new WantedModuleCoordinate(currentNetworkId, currentTransportStreamId, elementaryPid, moduleId, moduleVersion); if (wantedModuleCoordinates.Contains(coordinate)) return true; if (knownModuleCoordinates.Contains(coordinate)) return false; string combine = String.Join('/', "DSM-CC_Data", currentNetworkId.ToString(), currentTransportStreamId.ToString(), elementaryPid.ToString(), String.Format("{0}_V{1}.bin", moduleId, moduleVersion)); bool isWanted = !FileExists(combine); if (isWanted) { wantedModuleCoordinates.Add(coordinate); } else { knownModuleCoordinates.Add(coordinate); } return isWanted; } public bool TestForFramegrab(int currentNetworkId, int transportStreamId, ushort mappingProgramNumber, int mappingStreamElementaryPid) { string combine = string.Join('/', "Framegrabs", currentNetworkId.ToString(), transportStreamId.ToString(), String.Format("{1}_{0}.jpg", mappingStreamElementaryPid, mappingProgramNumber)); return FileExists(combine); } public void StoreFramegrab(int currentNetworkId, int transportStreamId, ushort mappingProgramNumber, ushort pid, byte[] imageData) { string combine = string.Join('/', "Framegrabs", currentNetworkId.ToString(), transportStreamId.ToString(), String.Format("{1}_{0}.jpg", pid, mappingProgramNumber)); WriteObject(combine, new MemoryStream(imageData), "image/jpeg"); } public void WaitForCompletion() { while (_tasks.Count > 0) { _tasks[0].Wait(); _tasks.RemoveAt(0); } wantedModuleCoordinates?.Clear(); knownModuleCoordinates?.Clear(); definetlyKnownFiles?.Clear(); } private HashSet wantedModuleCoordinates; private HashSet knownModuleCoordinates; private HashSet definetlyKnownFiles; struct WantedModuleCoordinate { private int currentNetworkId, currentTransportStreamId, elementaryPid; private ushort moduleId; private byte moduleVersion; public WantedModuleCoordinate(int currentNetworkId, int currentTransportStreamId, int elementaryPid, ushort moduleId, byte moduleVersion) { this.currentNetworkId = currentNetworkId; this.currentTransportStreamId = currentTransportStreamId; this.elementaryPid = elementaryPid; this.moduleId = moduleId; this.moduleVersion = moduleVersion; } public bool Equals(WantedModuleCoordinate other) { return currentNetworkId == other.currentNetworkId && currentTransportStreamId == other.currentTransportStreamId && elementaryPid == other.elementaryPid && moduleId == other.moduleId && moduleVersion == other.moduleVersion; } public override bool Equals(object obj) { return obj is WantedModuleCoordinate other && Equals(other); } public override int GetHashCode() { return HashCode.Combine(currentNetworkId, currentTransportStreamId, elementaryPid, moduleId, moduleVersion); } } private int uiVersion; private HashSet definetlyMissingFiles; public void UiSetVersion(int version) { this.uiVersion = version; } public object[] GetPluginConnector() { return new object[] { _minioClient, _minioBucket }; } public void Ping() { GetVersioningArgs args = new GetVersioningArgs().WithBucket(_minioBucket); VersioningConfiguration vc = _minioClient.GetVersioningAsync(args).Result; byte[] buffer = new byte[2048]; Random rng = new Random(); rng.NextBytes(buffer); string fname = String.Format("/test_write_{0}.bin", rng.NextInt64()); PutObjectArgs putObjectArgs = new PutObjectArgs(); putObjectArgs = putObjectArgs.WithBucket(_minioBucket); putObjectArgs = putObjectArgs.WithObject(fname); putObjectArgs = putObjectArgs.WithStreamData(new MemoryStream(buffer)); putObjectArgs = putObjectArgs.WithContentType("application/octet-stream"); putObjectArgs = putObjectArgs.WithObjectSize(2048); PutObjectResponse objectResponse = _minioClient.PutObjectAsync(putObjectArgs).Result; if (objectResponse.ResponseStatusCode != HttpStatusCode.OK) throw new Exception(objectResponse.ResponseContent); RemoveObjectArgs removeObjectArgs = new RemoveObjectArgs(); removeObjectArgs = removeObjectArgs.WithBucket(_minioBucket); removeObjectArgs = removeObjectArgs.WithObject(fname); Task removeObjectResponse = _minioClient.RemoveObjectAsync(removeObjectArgs); removeObjectResponse.Wait(); if (removeObjectResponse.Exception != null) throw removeObjectResponse.Exception; } public bool DvbNipTestForFile(string announcedFileContentLocation) { string path = "/nip/" + DvbNipUtilities.MakeFilename(announcedFileContentLocation); bool result = FileExists(path); return result; } public void DvbNipFileArrival(NipActualCarrierInformation carrier, FluteListener listener) { Dictionary bonusInfo = new Dictionary(); bonusInfo.Add("X-Skyscraper-NIP-StreamProviderName", carrier.NipStreamProviderName); bonusInfo.Add("X-Skyscraper-Event", nameof(DvbNipFileArrival)); bonusInfo.Add("X-Skyscraper-Content-Encoding",listener.FileAssociation.ContentEncoding); bonusInfo.Add("X-Skyscraper-Transfer-Length", listener.FileAssociation.TransferLength.ToString()); bonusInfo.Add("X-Skyscraper-Content-Length", listener.FileAssociation.ContentLength.ToString()); string mime = !string.IsNullOrEmpty(listener.FileAssociation.ContentType) ? listener.FileAssociation.ContentType : "application/octet-stream"; string path = "/nip/" + DvbNipUtilities.MakeFilename(listener.FileAssociation.ContentLocation); Stream stream = listener.ToStream(); WriteObject(path, stream, mime, bonusInfo); } public void StoreIqGraph(Guid jobGuid, long frequency, char polarity, IqChartData plot) { if (definetlyKnownFiles == null) definetlyKnownFiles = new HashSet(); if (definetlyMissingFiles == null) definetlyMissingFiles = new HashSet(); string fullName = String.Format("/iq/{0}/{1}_{2}.iq", jobGuid.ToString("D"),frequency,polarity); MemoryStream ms = new MemoryStream(); plot.SaveTo(ms); ms.Position = 0; WriteObject(fullName, ms); } public void StoreRfSpectrum(Guid jobGuid, RfSpectrumData rfSpectrum) { if (definetlyKnownFiles == null) definetlyKnownFiles = new HashSet(); if (definetlyMissingFiles == null) definetlyMissingFiles = new HashSet(); string fullName = String.Format("/rf/{0}.rf", jobGuid.ToString("D")); MemoryStream ms = new MemoryStream(); rfSpectrum.SaveTo(ms); ms.Position = 0; WriteObject(fullName, ms); } private bool DeleteFile(string fullname) { RemoveObjectArgs removeObject = new RemoveObjectArgs(); removeObject.WithObject(fullname); removeObject.WithBucket(_minioBucket); Task task = _minioClient.RemoveObjectAsync(removeObject); task.Wait(); if (task.Exception != null) return false; return true; } public void DeleteIqGraph(Guid selectedGuid, int frequencyItem1, SatelliteDeliverySystemDescriptor.PolarizationEnum frequencyItem2) { char polarityChar = frequencyItem2.ToString()[0]; string fullName = String.Format("/iq/{0}/{1}_{2}.iq", selectedGuid.ToString("D"), frequencyItem1, polarityChar); DeleteFile(fullName); } public void DeleteRfSpectrum(Guid selectedGuid) { string fullName = String.Format("/rf/{0}.rf", selectedGuid.ToString("D")); DeleteFile(fullName); } public bool OtvSsuTestFile(int? currentNetworkId, int? currentTransportStreamId, int sourcePid, ushort tableIdExtension, uint fileId, uint unknown1, uint length) { string cnid = currentNetworkId.HasValue ? currentNetworkId.Value.ToString() : "unknown"; string ctsid = currentTransportStreamId.HasValue ? currentTransportStreamId.Value.ToString() : "unknown"; string fname = tableIdExtension.ToString() + ".bin"; string path = String.Format("/OpenTV-SSU/{0}/{1}/{2}/{3}", cnid, ctsid, sourcePid.ToString(), fname); return FileExists(path); } public void OnOtvSsuComplete(int? currentNetworkId, int? currentTransportStreamId, int sourcePid, Stream getStream, ushort tableIdExtension, uint fileId, uint unknown1, uint length) { string cnid = currentNetworkId.HasValue ? currentNetworkId.Value.ToString() : "unknown"; string ctsid = currentTransportStreamId.HasValue ? currentTransportStreamId.Value.ToString() : "unknown"; string fname = tableIdExtension.ToString() + ".bin"; string path = String.Format("/OpenTV-SSU/{0}/{1}/{2}/{3}", cnid, ctsid, sourcePid.ToString(), fname); WriteObject(path, getStream); } public void OnNdsSsuComplete(int? currentNetworkId, int? currentTransportStreamId, int pid, ushort tableIdExtension, NdsSsuDataMap dataMap) { string cnid = currentNetworkId.HasValue ? currentNetworkId.Value.ToString() : "unknown"; string ctsid = currentTransportStreamId.HasValue ? currentTransportStreamId.Value.ToString() : "unknown"; string fname = tableIdExtension.ToString() + ".bin"; string path = String.Format("/NDS-SSU/{0}/{1}/{2}/{3}", cnid, ctsid, pid.ToString(), fname); long bufferLength = dataMap.CalculateLength(); MemoryStream buffer = new MemoryStream(new byte[bufferLength]); dataMap.WriteToStream(buffer); buffer.Position = 0; WriteObject(path, buffer); } public bool NdsSsuTestFile(int? currentNetworkId, int? currentTransportStreamId, int pid, ushort tableIdExtension) { string cnid = currentNetworkId.HasValue ? currentNetworkId.Value.ToString() : "unknown"; string ctsid = currentTransportStreamId.HasValue ? currentTransportStreamId.Value.ToString() : "unknown"; string fname = tableIdExtension.ToString() + ".bin"; string path = String.Format("/NDS-SSU/{0}/{1}/{2}/{3}", cnid, ctsid, pid.ToString(), fname); return FileExists(path); } public bool SsdpDeviceKnown(SsdpDevice ssdpDevice) { string filename = String.Format("/SSDP/{0}.xml", ssdpDevice.UniqueServiceName.SanitizeFileName()); return FileExists(filename); } public void SsdpStoreMetadata(SsdpDevice ssdpDevice, byte[] ssdpMetadataByteArray) { string filename = String.Format("/SSDP/{0}.xml", ssdpDevice.UniqueServiceName.SanitizeFileName()); WriteObject(filename, new MemoryStream(ssdpMetadataByteArray)); } public byte[] SsdpGetMetadata(SsdpDevice ssdpDevice) { string filename = String.Format("/SSDP/{0}.xml", ssdpDevice.UniqueServiceName.SanitizeFileName()); byte[] bytes = GetObject(filename); return bytes; } } }