2025-10-04 20:51:01 +02:00

520 lines
22 KiB
C#

using CommunityToolkit.HighPerformance.Buffers;
using Minio;
using Minio.DataModel;
using Minio.DataModel.Args;
using Minio.DataModel.Response;
using Minio.Exceptions;
using skyscraper5.Dvb.DataBroadcasting.SkyscraperVfs;
using skyscraper5.Dvb.Descriptors;
using skyscraper8.DvbNip;
using skyscraper8.Experimentals.NdsSsu;
using skyscraper8.Ietf.FLUTE;
using skyscraper8.Skyscraper.Drawing;
using skyscraper8.Skyscraper.Scraper.Storage;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Net;
using System.Runtime.InteropServices;
using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
using skyscraper5.Skyscraper;
using skyscraper8.SimpleServiceDiscoveryProtocol;
namespace skyscraper5.Data
{
public class MinioObjectStorage : ObjectStorage
{
private readonly IMinioClient _minioClient;
private readonly string _minioBucket;
private List<Task> _tasks;
private List<string> droppedFiles;
public MinioObjectStorage(IMinioClient minioClient, string minioBucket)
{
_minioClient = minioClient;
_minioBucket = minioBucket;
_tasks = new List<Task>();
droppedFiles = new List<string>();
}
private void WriteObject(string fullName, Stream buffer, string mime = "application/octet-stream", Dictionary<string, string> optionalData = null)
{
PutObjectArgs putObjectArgs = new PutObjectArgs();
putObjectArgs = putObjectArgs.WithBucket(_minioBucket);
putObjectArgs = putObjectArgs.WithObject(fullName);
putObjectArgs = putObjectArgs.WithStreamData(buffer);
//putObjectArgs = putObjectArgs.WithObjectSize(buffer.Length);
putObjectArgs = putObjectArgs.WithContentType(mime);
putObjectArgs = putObjectArgs.WithHeaders(optionalData);
if (buffer.CanSeek)
{
buffer.Position = 0;
putObjectArgs = putObjectArgs.WithObjectSize(buffer.Length);
}
else if (optionalData.ContainsKey("X-Skyscraper-Content-Length"))
{
putObjectArgs = putObjectArgs.WithObjectSize(long.Parse(optionalData["X-Skyscraper-Content-Length"]));
}
if (definetlyKnownFiles == null)
definetlyKnownFiles = new HashSet<string>();
if (definetlyMissingFiles == null)
definetlyMissingFiles = new HashSet<string>();
lock (_tasks)
{
_tasks.Add(_minioClient.PutObjectAsync(putObjectArgs).ContinueWith(task =>
{
if (task.IsFaulted)
{
AggregateException exception = task.Exception;
MinioException me = exception.InnerException as MinioException;
if (me != null)
{
if (me.ServerResponse == null)
throw new UnexpectedMinioException("Minio Server Response was null.");
System.Net.Http.HttpResponseMessage response = me.ServerResponse.Response;
if (response.StatusCode != HttpStatusCode.OK)
{
throw new MinioException("A minio upload task failed.", me);
}
}
else
throw new NotImplementedException();
}
droppedFiles.Add(fullName);
Monitor.Enter(definetlyKnownFiles);
definetlyKnownFiles.Add(fullName);
Monitor.Exit(definetlyKnownFiles);
Monitor.Enter(definetlyMissingFiles);
definetlyMissingFiles.Remove(fullName);
Monitor.Exit(definetlyMissingFiles);
buffer.Close();
buffer.Dispose();
}));
}
CleanTaskList();
}
private byte[] GetObject(string fullName)
{
StatObjectArgs statObjectArgs = new StatObjectArgs();
statObjectArgs.WithObject(fullName);
statObjectArgs.WithBucket(_minioBucket);
Task<ObjectStat> async = _minioClient.StatObjectAsync(statObjectArgs);
ObjectStat asyncResult = async.Result;
ObjectRetriever objectRetriever = new ObjectRetriever(asyncResult.Size);
GetObjectArgs getObjectArgs = new GetObjectArgs();
getObjectArgs.WithBucket(_minioBucket);
getObjectArgs = getObjectArgs.WithObject(fullName);
getObjectArgs.WithCallbackStream(objectRetriever.OurAction);
Task<ObjectStat> objectAsync = _minioClient.GetObjectAsync(getObjectArgs);
ObjectStat objectStat = objectAsync.Result;
return objectRetriever.GetBuffer();
}
public class ObjectRetriever
{
public ObjectRetriever(long size)
{
buffer = new byte[size];
}
private byte[] buffer;
public void OurAction(Stream obj)
{
int read = obj.Read(buffer, 0, (int)buffer.Length);
if (read != buffer.Length)
throw new MinioException("incomplete read");
}
public byte[] GetBuffer()
{
return buffer;
}
}
private void CleanTaskList()
{
while (true)
{
bool done = true;
lock (_tasks)
{
foreach (Task task in _tasks)
{
if (task.IsCompleted)
{
_tasks.Remove(task);
Debug.WriteLine(String.Format("Removed completed Task: {0}", task.ToString()));
done = false;
break;
}
}
}
if (done)
break;
}
}
public bool ObjectCarouselFileArrival(VfsFile vfsFile, int transportStreamId, int networkId)
{
string combine = string.Join('/', "DSM-CC_Objects", networkId.ToString(), transportStreamId.ToString(), vfsFile.SourcePid.ToString(), vfsFile.ToString().Substring(1));
combine = combine.Replace('\\', '/');
if (FileExists(combine))
return false;
Dictionary<string, string> extendedInfo = new Dictionary<string, string>();
extendedInfo.Add(nameof(vfsFile.ObjectLocation.CarouselId), vfsFile.ObjectLocation.CarouselId.ToString());
extendedInfo.Add(nameof(vfsFile.ObjectLocation.ModuleId), vfsFile.ObjectLocation.ModuleId.ToString());
extendedInfo.Add(nameof(vfsFile.ObjectLocation.Version), vfsFile.ObjectLocation.Version.ToString());
extendedInfo.Add(nameof(vfsFile.ObjectLocation.ObjectKey), BitConverter.ToString(vfsFile.ObjectLocation.ObjectKey));
WriteObject(combine,new MemoryStream(vfsFile.FileContent),"application/octet-stream",extendedInfo);
return true;
}
public void DataCarouselModuleArrival(int currentNetworkId, int currentTransportStreamId, int elementaryPid, ushort moduleModuleId, byte moduleModuleVersion, Stream result)
{
string combine = string.Join('/', "DSM-CC_Data", currentNetworkId.ToString(), currentTransportStreamId.ToString(), elementaryPid.ToString(), String.Format("{0}_V{1}.bin", moduleModuleId, moduleModuleVersion));
if (FileExists(combine))
return;
WriteObject(combine, result);
WantedModuleCoordinate coordinate = new WantedModuleCoordinate(currentNetworkId, currentTransportStreamId, elementaryPid, moduleModuleId, moduleModuleVersion);
knownModuleCoordinates.Add(coordinate);
wantedModuleCoordinates.Remove(coordinate);
}
public bool FileExists(string combine)
{
if (definetlyMissingFiles == null)
definetlyMissingFiles = new HashSet<string>();
if (definetlyKnownFiles == null)
definetlyKnownFiles = new HashSet<string>();
if (definetlyKnownFiles.Contains(combine))
return true;
if (definetlyMissingFiles.Contains(combine))
return false;
if (droppedFiles != null)
{
if (droppedFiles.Contains(combine))
return true;
}
StatObjectArgs soa = new StatObjectArgs().WithBucket(_minioBucket).WithObject(combine);
try
{
ObjectStat objectStat = _minioClient.StatObjectAsync(soa).Result;
definetlyKnownFiles.Add(combine);
return true;
}
catch (AggregateException e)
{
MinioException minioException = e.InnerExceptions[0] as MinioException;
ObjectNotFoundException objectNotFoundException = minioException as ObjectNotFoundException;
if (objectNotFoundException != null)
{
Monitor.Enter(definetlyMissingFiles);
definetlyMissingFiles.Add(combine);
Monitor.Exit(definetlyMissingFiles);
return false;
}
else
{
throw minioException;
}
}
}
public bool IsDsmCcModuleWanted(int currentNetworkId, int currentTransportStreamId, int elementaryPid, ushort moduleId, byte moduleVersion)
{
if (wantedModuleCoordinates == null)
wantedModuleCoordinates = new HashSet<WantedModuleCoordinate>();
if (knownModuleCoordinates == null)
knownModuleCoordinates = new HashSet<WantedModuleCoordinate>();
WantedModuleCoordinate coordinate = new WantedModuleCoordinate(currentNetworkId, currentTransportStreamId, elementaryPid, moduleId, moduleVersion);
if (wantedModuleCoordinates.Contains(coordinate))
return true;
if (knownModuleCoordinates.Contains(coordinate))
return false;
string combine = String.Join('/', "DSM-CC_Data", currentNetworkId.ToString(), currentTransportStreamId.ToString(), elementaryPid.ToString(), String.Format("{0}_V{1}.bin", moduleId, moduleVersion));
bool isWanted = !FileExists(combine);
if (isWanted)
{
wantedModuleCoordinates.Add(coordinate);
}
else
{
knownModuleCoordinates.Add(coordinate);
}
return isWanted;
}
public bool TestForFramegrab(int currentNetworkId, int transportStreamId, ushort mappingProgramNumber, int mappingStreamElementaryPid)
{
string combine = string.Join('/', "Framegrabs", currentNetworkId.ToString(), transportStreamId.ToString(), String.Format("{1}_{0}.jpg", mappingStreamElementaryPid, mappingProgramNumber));
return FileExists(combine);
}
public void StoreFramegrab(int currentNetworkId, int transportStreamId, ushort mappingProgramNumber, ushort pid, byte[] imageData)
{
string combine = string.Join('/', "Framegrabs", currentNetworkId.ToString(), transportStreamId.ToString(), String.Format("{1}_{0}.jpg", pid, mappingProgramNumber));
WriteObject(combine, new MemoryStream(imageData), "image/jpeg");
}
public void WaitForCompletion()
{
while (_tasks.Count > 0)
{
_tasks[0].Wait();
_tasks.RemoveAt(0);
}
wantedModuleCoordinates?.Clear();
knownModuleCoordinates?.Clear();
definetlyKnownFiles?.Clear();
}
private HashSet<WantedModuleCoordinate> wantedModuleCoordinates;
private HashSet<WantedModuleCoordinate> knownModuleCoordinates;
private HashSet<string> definetlyKnownFiles;
struct WantedModuleCoordinate
{
private int currentNetworkId, currentTransportStreamId, elementaryPid;
private ushort moduleId;
private byte moduleVersion;
public WantedModuleCoordinate(int currentNetworkId, int currentTransportStreamId, int elementaryPid, ushort moduleId, byte moduleVersion)
{
this.currentNetworkId = currentNetworkId;
this.currentTransportStreamId = currentTransportStreamId;
this.elementaryPid = elementaryPid;
this.moduleId = moduleId;
this.moduleVersion = moduleVersion;
}
public bool Equals(WantedModuleCoordinate other)
{
return currentNetworkId == other.currentNetworkId && currentTransportStreamId == other.currentTransportStreamId && elementaryPid == other.elementaryPid && moduleId == other.moduleId && moduleVersion == other.moduleVersion;
}
public override bool Equals(object obj)
{
return obj is WantedModuleCoordinate other && Equals(other);
}
public override int GetHashCode()
{
return HashCode.Combine(currentNetworkId, currentTransportStreamId, elementaryPid, moduleId, moduleVersion);
}
}
private int uiVersion;
private HashSet<string> definetlyMissingFiles;
public void UiSetVersion(int version)
{
this.uiVersion = version;
}
public object[] GetPluginConnector()
{
return new object[] { _minioClient, _minioBucket };
}
public void Ping()
{
GetVersioningArgs args = new GetVersioningArgs().WithBucket(_minioBucket);
VersioningConfiguration vc = _minioClient.GetVersioningAsync(args).Result;
byte[] buffer = new byte[2048];
Random rng = new Random();
rng.NextBytes(buffer);
string fname = String.Format("/test_write_{0}.bin", rng.NextInt64());
PutObjectArgs putObjectArgs = new PutObjectArgs();
putObjectArgs = putObjectArgs.WithBucket(_minioBucket);
putObjectArgs = putObjectArgs.WithObject(fname);
putObjectArgs = putObjectArgs.WithStreamData(new MemoryStream(buffer));
putObjectArgs = putObjectArgs.WithContentType("application/octet-stream");
putObjectArgs = putObjectArgs.WithObjectSize(2048);
PutObjectResponse objectResponse = _minioClient.PutObjectAsync(putObjectArgs).Result;
if (objectResponse.ResponseStatusCode != HttpStatusCode.OK)
throw new Exception(objectResponse.ResponseContent);
RemoveObjectArgs removeObjectArgs = new RemoveObjectArgs();
removeObjectArgs = removeObjectArgs.WithBucket(_minioBucket);
removeObjectArgs = removeObjectArgs.WithObject(fname);
Task removeObjectResponse = _minioClient.RemoveObjectAsync(removeObjectArgs);
removeObjectResponse.Wait();
if (removeObjectResponse.Exception != null)
throw removeObjectResponse.Exception;
}
public bool DvbNipTestForFile(string announcedFileContentLocation)
{
string path = "/nip/" + DvbNipUtilities.MakeFilename(announcedFileContentLocation);
bool result = FileExists(path);
return result;
}
public void DvbNipFileArrival(NipActualCarrierInformation carrier, FluteListener listener)
{
Dictionary<string, string> bonusInfo = new Dictionary<string, string>();
bonusInfo.Add("X-Skyscraper-NIP-StreamProviderName", carrier.NipStreamProviderName);
bonusInfo.Add("X-Skyscraper-Event", nameof(DvbNipFileArrival));
bonusInfo.Add("X-Skyscraper-Content-Encoding",listener.FileAssociation.ContentEncoding);
bonusInfo.Add("X-Skyscraper-Transfer-Length", listener.FileAssociation.TransferLength.ToString());
bonusInfo.Add("X-Skyscraper-Content-Length", listener.FileAssociation.ContentLength.ToString());
string mime = !string.IsNullOrEmpty(listener.FileAssociation.ContentType)
? listener.FileAssociation.ContentType
: "application/octet-stream";
string path = "/nip/" + DvbNipUtilities.MakeFilename(listener.FileAssociation.ContentLocation);
Stream stream = listener.ToStream();
WriteObject(path, stream, mime, bonusInfo);
}
public void StoreIqGraph(Guid jobGuid, long frequency, char polarity, IqChartData plot)
{
if (definetlyKnownFiles == null)
definetlyKnownFiles = new HashSet<string>();
if (definetlyMissingFiles == null)
definetlyMissingFiles = new HashSet<string>();
string fullName = String.Format("/iq/{0}/{1}_{2}.iq", jobGuid.ToString("D"),frequency,polarity);
MemoryStream ms = new MemoryStream();
plot.SaveTo(ms);
ms.Position = 0;
WriteObject(fullName, ms);
}
public void StoreRfSpectrum(Guid jobGuid, RfSpectrumData rfSpectrum)
{
if (definetlyKnownFiles == null)
definetlyKnownFiles = new HashSet<string>();
if (definetlyMissingFiles == null)
definetlyMissingFiles = new HashSet<string>();
string fullName = String.Format("/rf/{0}.rf", jobGuid.ToString("D"));
MemoryStream ms = new MemoryStream();
rfSpectrum.SaveTo(ms);
ms.Position = 0;
WriteObject(fullName, ms);
}
private bool DeleteFile(string fullname)
{
RemoveObjectArgs removeObject = new RemoveObjectArgs();
removeObject.WithObject(fullname);
removeObject.WithBucket(_minioBucket);
Task task = _minioClient.RemoveObjectAsync(removeObject);
task.Wait();
if (task.Exception != null)
return false;
return true;
}
public void DeleteIqGraph(Guid selectedGuid, int frequencyItem1, SatelliteDeliverySystemDescriptor.PolarizationEnum frequencyItem2)
{
char polarityChar = frequencyItem2.ToString()[0];
string fullName = String.Format("/iq/{0}/{1}_{2}.iq", selectedGuid.ToString("D"), frequencyItem1, polarityChar);
DeleteFile(fullName);
}
public void DeleteRfSpectrum(Guid selectedGuid)
{
string fullName = String.Format("/rf/{0}.rf", selectedGuid.ToString("D"));
DeleteFile(fullName);
}
public bool OtvSsuTestFile(int? currentNetworkId, int? currentTransportStreamId, int sourcePid, ushort tableIdExtension,
uint fileId, uint unknown1, uint length)
{
string cnid = currentNetworkId.HasValue ? currentNetworkId.Value.ToString() : "unknown";
string ctsid = currentTransportStreamId.HasValue ? currentTransportStreamId.Value.ToString() : "unknown";
string fname = tableIdExtension.ToString() + ".bin";
string path = String.Format("/OpenTV-SSU/{0}/{1}/{2}/{3}", cnid, ctsid, sourcePid.ToString(), fname);
return FileExists(path);
}
public void OnOtvSsuComplete(int? currentNetworkId, int? currentTransportStreamId, int sourcePid, Stream getStream,
ushort tableIdExtension, uint fileId, uint unknown1, uint length)
{
string cnid = currentNetworkId.HasValue ? currentNetworkId.Value.ToString() : "unknown";
string ctsid = currentTransportStreamId.HasValue ? currentTransportStreamId.Value.ToString() : "unknown";
string fname = tableIdExtension.ToString() + ".bin";
string path = String.Format("/OpenTV-SSU/{0}/{1}/{2}/{3}", cnid, ctsid, sourcePid.ToString(), fname);
WriteObject(path, getStream);
}
public void OnNdsSsuComplete(int? currentNetworkId, int? currentTransportStreamId, int pid, ushort tableIdExtension,
NdsSsuDataMap dataMap)
{
string cnid = currentNetworkId.HasValue ? currentNetworkId.Value.ToString() : "unknown";
string ctsid = currentTransportStreamId.HasValue ? currentTransportStreamId.Value.ToString() : "unknown";
string fname = tableIdExtension.ToString() + ".bin";
string path = String.Format("/NDS-SSU/{0}/{1}/{2}/{3}", cnid, ctsid, pid.ToString(), fname);
long bufferLength = dataMap.CalculateLength();
MemoryStream buffer = new MemoryStream(new byte[bufferLength]);
dataMap.WriteToStream(buffer);
buffer.Position = 0;
WriteObject(path, buffer);
}
public bool NdsSsuTestFile(int? currentNetworkId, int? currentTransportStreamId, int pid, ushort tableIdExtension)
{
string cnid = currentNetworkId.HasValue ? currentNetworkId.Value.ToString() : "unknown";
string ctsid = currentTransportStreamId.HasValue ? currentTransportStreamId.Value.ToString() : "unknown";
string fname = tableIdExtension.ToString() + ".bin";
string path = String.Format("/NDS-SSU/{0}/{1}/{2}/{3}", cnid, ctsid, pid.ToString(), fname);
return FileExists(path);
}
public bool SsdpDeviceKnown(SsdpDevice ssdpDevice)
{
string filename = String.Format("/SSDP/{0}.xml", ssdpDevice.UniqueServiceName.SanitizeFileName());
return FileExists(filename);
}
public void SsdpStoreMetadata(SsdpDevice ssdpDevice, byte[] ssdpMetadataByteArray)
{
string filename = String.Format("/SSDP/{0}.xml", ssdpDevice.UniqueServiceName.SanitizeFileName());
WriteObject(filename, new MemoryStream(ssdpMetadataByteArray));
}
public byte[] SsdpGetMetadata(SsdpDevice ssdpDevice)
{
string filename = String.Format("/SSDP/{0}.xml", ssdpDevice.UniqueServiceName.SanitizeFileName());
byte[] bytes = GetObject(filename);
return bytes;
}
}
}