506 lines
21 KiB
C#
506 lines
21 KiB
C#
using CommunityToolkit.HighPerformance.Buffers;
|
|
using Minio;
|
|
using Minio.DataModel;
|
|
using Minio.DataModel.Args;
|
|
using Minio.DataModel.Response;
|
|
using Minio.Exceptions;
|
|
using skyscraper5.Dvb.DataBroadcasting.SkyscraperVfs;
|
|
using skyscraper5.Dvb.Descriptors;
|
|
using skyscraper8.DvbNip;
|
|
using skyscraper8.Experimentals.NdsSsu;
|
|
using skyscraper8.Ietf.FLUTE;
|
|
using skyscraper8.Skyscraper.Drawing;
|
|
using skyscraper8.Skyscraper.Scraper.Storage;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Diagnostics;
|
|
using System.Globalization;
|
|
using System.IO;
|
|
using System.Net;
|
|
using System.Runtime.InteropServices;
|
|
using System.Security.Cryptography;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using skyscraper5.Skyscraper;
|
|
using skyscraper8.SimpleServiceDiscoveryProtocol;
|
|
|
|
namespace skyscraper5.Data
|
|
{
|
|
public class MinioObjectStorage : ObjectStorage
|
|
{
|
|
private readonly IMinioClient _minioClient;
|
|
private readonly string _minioBucket;
|
|
private List<Task> _tasks;
|
|
private List<string> droppedFiles;
|
|
|
|
public MinioObjectStorage(IMinioClient minioClient, string minioBucket)
|
|
{
|
|
_minioClient = minioClient;
|
|
_minioBucket = minioBucket;
|
|
_tasks = new List<Task>();
|
|
droppedFiles = new List<string>();
|
|
}
|
|
|
|
private void WriteObject(string fullName, Stream buffer, string mime = "application/octet-stream", Dictionary<string, string> optionalData = null)
|
|
{
|
|
PutObjectArgs putObjectArgs = new PutObjectArgs();
|
|
putObjectArgs = putObjectArgs.WithBucket(_minioBucket);
|
|
putObjectArgs = putObjectArgs.WithObject(fullName);
|
|
putObjectArgs = putObjectArgs.WithStreamData(buffer);
|
|
//putObjectArgs = putObjectArgs.WithObjectSize(buffer.Length);
|
|
putObjectArgs = putObjectArgs.WithContentType(mime);
|
|
putObjectArgs = putObjectArgs.WithHeaders(optionalData);
|
|
if (buffer.CanSeek)
|
|
{
|
|
putObjectArgs = putObjectArgs.WithObjectSize(buffer.Length);
|
|
}
|
|
else if (optionalData.ContainsKey("X-Skyscraper-Content-Length"))
|
|
{
|
|
putObjectArgs = putObjectArgs.WithObjectSize(long.Parse(optionalData["X-Skyscraper-Content-Length"]));
|
|
}
|
|
|
|
if (definetlyKnownFiles == null)
|
|
definetlyKnownFiles = new HashSet<string>();
|
|
|
|
if (definetlyMissingFiles == null)
|
|
definetlyMissingFiles = new HashSet<string>();
|
|
|
|
|
|
lock (_tasks)
|
|
{
|
|
_tasks.Add(_minioClient.PutObjectAsync(putObjectArgs).ContinueWith(task =>
|
|
{
|
|
if (task.IsFaulted)
|
|
{
|
|
throw new MinioException("A minio upload task failed.");
|
|
}
|
|
droppedFiles.Add(fullName);
|
|
|
|
Monitor.Enter(definetlyKnownFiles);
|
|
definetlyKnownFiles.Add(fullName);
|
|
Monitor.Exit(definetlyKnownFiles);
|
|
|
|
Monitor.Enter(definetlyMissingFiles);
|
|
definetlyMissingFiles.Remove(fullName);
|
|
Monitor.Exit(definetlyMissingFiles);
|
|
|
|
buffer.Close();
|
|
buffer.Dispose();
|
|
}));
|
|
}
|
|
|
|
CleanTaskList();
|
|
}
|
|
|
|
private byte[] GetObject(string fullName)
|
|
{
|
|
StatObjectArgs statObjectArgs = new StatObjectArgs();
|
|
statObjectArgs.WithObject(fullName);
|
|
statObjectArgs.WithBucket(_minioBucket);
|
|
Task<ObjectStat> async = _minioClient.StatObjectAsync(statObjectArgs);
|
|
ObjectStat asyncResult = async.Result;
|
|
|
|
ObjectRetriever objectRetriever = new ObjectRetriever(asyncResult.Size);
|
|
GetObjectArgs getObjectArgs = new GetObjectArgs();
|
|
getObjectArgs.WithBucket(_minioBucket);
|
|
getObjectArgs = getObjectArgs.WithObject(fullName);
|
|
getObjectArgs.WithCallbackStream(objectRetriever.OurAction);
|
|
Task<ObjectStat> objectAsync = _minioClient.GetObjectAsync(getObjectArgs);
|
|
ObjectStat objectStat = objectAsync.Result;
|
|
return objectRetriever.GetBuffer();
|
|
}
|
|
|
|
public class ObjectRetriever
|
|
{
|
|
public ObjectRetriever(long size)
|
|
{
|
|
buffer = new byte[size];
|
|
}
|
|
|
|
private byte[] buffer;
|
|
public void OurAction(Stream obj)
|
|
{
|
|
int read = obj.Read(buffer, 0, (int)buffer.Length);
|
|
if (read != buffer.Length)
|
|
throw new MinioException("incomplete read");
|
|
}
|
|
|
|
public byte[] GetBuffer()
|
|
{
|
|
return buffer;
|
|
}
|
|
}
|
|
|
|
private void CleanTaskList()
|
|
{
|
|
while (true)
|
|
{
|
|
bool done = true;
|
|
lock (_tasks)
|
|
{
|
|
foreach (Task task in _tasks)
|
|
{
|
|
if (task.IsCompleted)
|
|
{
|
|
_tasks.Remove(task);
|
|
Debug.WriteLine(String.Format("Removed completed Task: {0}", task.ToString()));
|
|
done = false;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (done)
|
|
break;
|
|
}
|
|
}
|
|
|
|
public bool ObjectCarouselFileArrival(VfsFile vfsFile, int transportStreamId, int networkId)
|
|
{
|
|
string combine = string.Join('/', "DSM-CC_Objects", networkId.ToString(), transportStreamId.ToString(), vfsFile.SourcePid.ToString(), vfsFile.ToString().Substring(1));
|
|
combine = combine.Replace('\\', '/');
|
|
if (FileExists(combine))
|
|
return false;
|
|
|
|
Dictionary<string, string> extendedInfo = new Dictionary<string, string>();
|
|
extendedInfo.Add(nameof(vfsFile.ObjectLocation.CarouselId), vfsFile.ObjectLocation.CarouselId.ToString());
|
|
extendedInfo.Add(nameof(vfsFile.ObjectLocation.ModuleId), vfsFile.ObjectLocation.ModuleId.ToString());
|
|
extendedInfo.Add(nameof(vfsFile.ObjectLocation.Version), vfsFile.ObjectLocation.Version.ToString());
|
|
extendedInfo.Add(nameof(vfsFile.ObjectLocation.ObjectKey), BitConverter.ToString(vfsFile.ObjectLocation.ObjectKey));
|
|
|
|
WriteObject(combine,new MemoryStream(vfsFile.FileContent),"application/octet-stream",extendedInfo);
|
|
return true;
|
|
}
|
|
|
|
public void DataCarouselModuleArrival(int currentNetworkId, int currentTransportStreamId, int elementaryPid, ushort moduleModuleId, byte moduleModuleVersion, Stream result)
|
|
{
|
|
string combine = string.Join('/', "DSM-CC_Data", currentNetworkId.ToString(), currentTransportStreamId.ToString(), elementaryPid.ToString(), String.Format("{0}_V{1}.bin", moduleModuleId, moduleModuleVersion));
|
|
if (FileExists(combine))
|
|
return;
|
|
|
|
WriteObject(combine, result);
|
|
|
|
WantedModuleCoordinate coordinate = new WantedModuleCoordinate(currentNetworkId, currentTransportStreamId, elementaryPid, moduleModuleId, moduleModuleVersion);
|
|
knownModuleCoordinates.Add(coordinate);
|
|
wantedModuleCoordinates.Remove(coordinate);
|
|
}
|
|
|
|
public bool FileExists(string combine)
|
|
{
|
|
if (definetlyMissingFiles == null)
|
|
definetlyMissingFiles = new HashSet<string>();
|
|
if (definetlyKnownFiles == null)
|
|
definetlyKnownFiles = new HashSet<string>();
|
|
|
|
if (definetlyKnownFiles.Contains(combine))
|
|
return true;
|
|
if (definetlyMissingFiles.Contains(combine))
|
|
return false;
|
|
|
|
if (droppedFiles != null)
|
|
{
|
|
if (droppedFiles.Contains(combine))
|
|
return true;
|
|
}
|
|
|
|
StatObjectArgs soa = new StatObjectArgs().WithBucket(_minioBucket).WithObject(combine);
|
|
try
|
|
{
|
|
ObjectStat objectStat = _minioClient.StatObjectAsync(soa).Result;
|
|
definetlyKnownFiles.Add(combine);
|
|
return true;
|
|
}
|
|
catch (AggregateException e)
|
|
{
|
|
MinioException minioException = e.InnerExceptions[0] as MinioException;
|
|
ObjectNotFoundException objectNotFoundException = minioException as ObjectNotFoundException;
|
|
if (objectNotFoundException != null)
|
|
{
|
|
|
|
Monitor.Enter(definetlyMissingFiles);
|
|
definetlyMissingFiles.Add(combine);
|
|
Monitor.Exit(definetlyMissingFiles);
|
|
|
|
return false;
|
|
}
|
|
else
|
|
{
|
|
throw minioException;
|
|
}
|
|
}
|
|
}
|
|
|
|
public bool IsDsmCcModuleWanted(int currentNetworkId, int currentTransportStreamId, int elementaryPid, ushort moduleId, byte moduleVersion)
|
|
{
|
|
if (wantedModuleCoordinates == null)
|
|
wantedModuleCoordinates = new HashSet<WantedModuleCoordinate>();
|
|
if (knownModuleCoordinates == null)
|
|
knownModuleCoordinates = new HashSet<WantedModuleCoordinate>();
|
|
|
|
WantedModuleCoordinate coordinate = new WantedModuleCoordinate(currentNetworkId, currentTransportStreamId, elementaryPid, moduleId, moduleVersion);
|
|
if (wantedModuleCoordinates.Contains(coordinate))
|
|
return true;
|
|
if (knownModuleCoordinates.Contains(coordinate))
|
|
return false;
|
|
|
|
string combine = String.Join('/', "DSM-CC_Data", currentNetworkId.ToString(), currentTransportStreamId.ToString(), elementaryPid.ToString(), String.Format("{0}_V{1}.bin", moduleId, moduleVersion));
|
|
bool isWanted = !FileExists(combine);
|
|
if (isWanted)
|
|
{
|
|
wantedModuleCoordinates.Add(coordinate);
|
|
}
|
|
else
|
|
{
|
|
knownModuleCoordinates.Add(coordinate);
|
|
}
|
|
return isWanted;
|
|
}
|
|
|
|
public bool TestForFramegrab(int currentNetworkId, int transportStreamId, ushort mappingProgramNumber, int mappingStreamElementaryPid)
|
|
{
|
|
string combine = string.Join('/', "Framegrabs", currentNetworkId.ToString(), transportStreamId.ToString(), String.Format("{1}_{0}.jpg", mappingStreamElementaryPid, mappingProgramNumber));
|
|
return FileExists(combine);
|
|
}
|
|
|
|
public void StoreFramegrab(int currentNetworkId, int transportStreamId, ushort mappingProgramNumber, ushort pid, byte[] imageData)
|
|
{
|
|
string combine = string.Join('/', "Framegrabs", currentNetworkId.ToString(), transportStreamId.ToString(), String.Format("{1}_{0}.jpg", pid, mappingProgramNumber));
|
|
WriteObject(combine, new MemoryStream(imageData), "image/jpeg");
|
|
}
|
|
|
|
public void WaitForCompletion()
|
|
{
|
|
while (_tasks.Count > 0)
|
|
{
|
|
_tasks[0].Wait();
|
|
_tasks.RemoveAt(0);
|
|
}
|
|
wantedModuleCoordinates?.Clear();
|
|
knownModuleCoordinates?.Clear();
|
|
definetlyKnownFiles?.Clear();
|
|
}
|
|
|
|
private HashSet<WantedModuleCoordinate> wantedModuleCoordinates;
|
|
private HashSet<WantedModuleCoordinate> knownModuleCoordinates;
|
|
private HashSet<string> definetlyKnownFiles;
|
|
struct WantedModuleCoordinate
|
|
{
|
|
private int currentNetworkId, currentTransportStreamId, elementaryPid;
|
|
private ushort moduleId;
|
|
private byte moduleVersion;
|
|
|
|
public WantedModuleCoordinate(int currentNetworkId, int currentTransportStreamId, int elementaryPid, ushort moduleId, byte moduleVersion)
|
|
{
|
|
this.currentNetworkId = currentNetworkId;
|
|
this.currentTransportStreamId = currentTransportStreamId;
|
|
this.elementaryPid = elementaryPid;
|
|
this.moduleId = moduleId;
|
|
this.moduleVersion = moduleVersion;
|
|
}
|
|
|
|
public bool Equals(WantedModuleCoordinate other)
|
|
{
|
|
return currentNetworkId == other.currentNetworkId && currentTransportStreamId == other.currentTransportStreamId && elementaryPid == other.elementaryPid && moduleId == other.moduleId && moduleVersion == other.moduleVersion;
|
|
}
|
|
|
|
public override bool Equals(object obj)
|
|
{
|
|
return obj is WantedModuleCoordinate other && Equals(other);
|
|
}
|
|
|
|
public override int GetHashCode()
|
|
{
|
|
return HashCode.Combine(currentNetworkId, currentTransportStreamId, elementaryPid, moduleId, moduleVersion);
|
|
}
|
|
}
|
|
|
|
private int uiVersion;
|
|
private HashSet<string> definetlyMissingFiles;
|
|
|
|
public void UiSetVersion(int version)
|
|
{
|
|
this.uiVersion = version;
|
|
}
|
|
|
|
public object[] GetPluginConnector()
|
|
{
|
|
return new object[] { _minioClient, _minioBucket };
|
|
}
|
|
|
|
public void Ping()
|
|
{
|
|
GetVersioningArgs args = new GetVersioningArgs().WithBucket(_minioBucket);
|
|
VersioningConfiguration vc = _minioClient.GetVersioningAsync(args).Result;
|
|
|
|
byte[] buffer = new byte[2048];
|
|
Random rng = new Random();
|
|
rng.NextBytes(buffer);
|
|
string fname = String.Format("/test_write_{0}.bin", rng.NextInt64());
|
|
|
|
PutObjectArgs putObjectArgs = new PutObjectArgs();
|
|
putObjectArgs = putObjectArgs.WithBucket(_minioBucket);
|
|
putObjectArgs = putObjectArgs.WithObject(fname);
|
|
putObjectArgs = putObjectArgs.WithStreamData(new MemoryStream(buffer));
|
|
putObjectArgs = putObjectArgs.WithContentType("application/octet-stream");
|
|
putObjectArgs = putObjectArgs.WithObjectSize(2048);
|
|
|
|
PutObjectResponse objectResponse = _minioClient.PutObjectAsync(putObjectArgs).Result;
|
|
if (objectResponse.ResponseStatusCode != HttpStatusCode.OK)
|
|
throw new Exception(objectResponse.ResponseContent);
|
|
|
|
RemoveObjectArgs removeObjectArgs = new RemoveObjectArgs();
|
|
removeObjectArgs = removeObjectArgs.WithBucket(_minioBucket);
|
|
removeObjectArgs = removeObjectArgs.WithObject(fname);
|
|
|
|
Task removeObjectResponse = _minioClient.RemoveObjectAsync(removeObjectArgs);
|
|
removeObjectResponse.Wait();
|
|
if (removeObjectResponse.Exception != null)
|
|
throw removeObjectResponse.Exception;
|
|
}
|
|
|
|
public bool DvbNipTestForFile(string announcedFileContentLocation)
|
|
{
|
|
string path = "/nip/" + DvbNipUtilities.MakeFilename(announcedFileContentLocation);
|
|
bool result = FileExists(path);
|
|
return result;
|
|
}
|
|
|
|
public void DvbNipFileArrival(NipActualCarrierInformation carrier, FluteListener listener)
|
|
{
|
|
Dictionary<string, string> bonusInfo = new Dictionary<string, string>();
|
|
bonusInfo.Add("X-Skyscraper-NIP-StreamProviderName", carrier.NipStreamProviderName);
|
|
bonusInfo.Add("X-Skyscraper-Event", nameof(DvbNipFileArrival));
|
|
bonusInfo.Add("X-Skyscraper-Content-Encoding",listener.FileAssociation.ContentEncoding);
|
|
bonusInfo.Add("X-Skyscraper-Transfer-Length", listener.FileAssociation.TransferLength.ToString());
|
|
bonusInfo.Add("X-Skyscraper-Content-Length", listener.FileAssociation.ContentLength.ToString());
|
|
|
|
string mime = !string.IsNullOrEmpty(listener.FileAssociation.ContentType)
|
|
? listener.FileAssociation.ContentType
|
|
: "application/octet-stream";
|
|
|
|
string path = "/nip/" + DvbNipUtilities.MakeFilename(listener.FileAssociation.ContentLocation);
|
|
Stream stream = listener.ToStream();
|
|
WriteObject(path, stream, mime, bonusInfo);
|
|
}
|
|
|
|
public void StoreIqGraph(Guid jobGuid, long frequency, char polarity, IqChartData plot)
|
|
{
|
|
if (definetlyKnownFiles == null)
|
|
definetlyKnownFiles = new HashSet<string>();
|
|
if (definetlyMissingFiles == null)
|
|
definetlyMissingFiles = new HashSet<string>();
|
|
|
|
string fullName = String.Format("/iq/{0}/{1}_{2}.iq", jobGuid.ToString("D"),frequency,polarity);
|
|
MemoryStream ms = new MemoryStream();
|
|
plot.SaveTo(ms);
|
|
ms.Position = 0;
|
|
|
|
WriteObject(fullName, ms);
|
|
}
|
|
|
|
public void StoreRfSpectrum(Guid jobGuid, RfSpectrumData rfSpectrum)
|
|
{
|
|
if (definetlyKnownFiles == null)
|
|
definetlyKnownFiles = new HashSet<string>();
|
|
if (definetlyMissingFiles == null)
|
|
definetlyMissingFiles = new HashSet<string>();
|
|
|
|
string fullName = String.Format("/rf/{0}.rf", jobGuid.ToString("D"));
|
|
MemoryStream ms = new MemoryStream();
|
|
rfSpectrum.SaveTo(ms);
|
|
ms.Position = 0;
|
|
|
|
WriteObject(fullName, ms);
|
|
}
|
|
|
|
private bool DeleteFile(string fullname)
|
|
{
|
|
RemoveObjectArgs removeObject = new RemoveObjectArgs();
|
|
removeObject.WithObject(fullname);
|
|
removeObject.WithBucket(_minioBucket);
|
|
Task task = _minioClient.RemoveObjectAsync(removeObject);
|
|
task.Wait();
|
|
if (task.Exception != null)
|
|
return false;
|
|
return true;
|
|
}
|
|
|
|
public void DeleteIqGraph(Guid selectedGuid, int frequencyItem1, SatelliteDeliverySystemDescriptor.PolarizationEnum frequencyItem2)
|
|
{
|
|
char polarityChar = frequencyItem2.ToString()[0];
|
|
string fullName = String.Format("/iq/{0}/{1}_{2}.iq", selectedGuid.ToString("D"), frequencyItem1, polarityChar);
|
|
DeleteFile(fullName);
|
|
}
|
|
|
|
public void DeleteRfSpectrum(Guid selectedGuid)
|
|
{
|
|
string fullName = String.Format("/rf/{0}.rf", selectedGuid.ToString("D"));
|
|
DeleteFile(fullName);
|
|
}
|
|
|
|
public bool OtvSsuTestFile(int? currentNetworkId, int? currentTransportStreamId, int sourcePid, ushort tableIdExtension,
|
|
uint fileId, uint unknown1, uint length)
|
|
{
|
|
string cnid = currentNetworkId.HasValue ? currentNetworkId.Value.ToString() : "unknown";
|
|
string ctsid = currentTransportStreamId.HasValue ? currentTransportStreamId.Value.ToString() : "unknown";
|
|
string fname = tableIdExtension.ToString() + ".bin";
|
|
string path = String.Format("/OpenTV-SSU/{0}/{1}/{2}/{3}", cnid, ctsid, sourcePid.ToString(), fname);
|
|
return FileExists(path);
|
|
}
|
|
|
|
public void OnOtvSsuComplete(int? currentNetworkId, int? currentTransportStreamId, int sourcePid, Stream getStream,
|
|
ushort tableIdExtension, uint fileId, uint unknown1, uint length)
|
|
{
|
|
string cnid = currentNetworkId.HasValue ? currentNetworkId.Value.ToString() : "unknown";
|
|
string ctsid = currentTransportStreamId.HasValue ? currentTransportStreamId.Value.ToString() : "unknown";
|
|
string fname = tableIdExtension.ToString() + ".bin";
|
|
string path = String.Format("/OpenTV-SSU/{0}/{1}/{2}/{3}", cnid, ctsid, sourcePid.ToString(), fname);
|
|
|
|
WriteObject(path, getStream);
|
|
}
|
|
|
|
public void OnNdsSsuComplete(int? currentNetworkId, int? currentTransportStreamId, int pid, ushort tableIdExtension,
|
|
NdsSsuDataMap dataMap)
|
|
{
|
|
string cnid = currentNetworkId.HasValue ? currentNetworkId.Value.ToString() : "unknown";
|
|
string ctsid = currentTransportStreamId.HasValue ? currentTransportStreamId.Value.ToString() : "unknown";
|
|
string fname = tableIdExtension.ToString() + ".bin";
|
|
string path = String.Format("/NDS-SSU/{0}/{1}/{2}/{3}", cnid, ctsid, pid.ToString(), fname);
|
|
|
|
long bufferLength = dataMap.CalculateLength();
|
|
MemoryStream buffer = new MemoryStream(new byte[bufferLength]);
|
|
dataMap.WriteToStream(buffer);
|
|
buffer.Position = 0;
|
|
WriteObject(path, buffer);
|
|
}
|
|
|
|
public bool NdsSsuTestFile(int? currentNetworkId, int? currentTransportStreamId, int pid, ushort tableIdExtension)
|
|
{
|
|
string cnid = currentNetworkId.HasValue ? currentNetworkId.Value.ToString() : "unknown";
|
|
string ctsid = currentTransportStreamId.HasValue ? currentTransportStreamId.Value.ToString() : "unknown";
|
|
string fname = tableIdExtension.ToString() + ".bin";
|
|
string path = String.Format("/NDS-SSU/{0}/{1}/{2}/{3}", cnid, ctsid, pid.ToString(), fname);
|
|
return FileExists(path);
|
|
}
|
|
|
|
public bool SsdpDeviceKnown(SsdpDevice ssdpDevice)
|
|
{
|
|
string filename = String.Format("/SSDP/{0}.xml", ssdpDevice.UniqueServiceName.SanitizeFileName());
|
|
return FileExists(filename);
|
|
}
|
|
|
|
public void SsdpStoreMetadata(SsdpDevice ssdpDevice, byte[] ssdpMetadataByteArray)
|
|
{
|
|
string filename = String.Format("/SSDP/{0}.xml", ssdpDevice.UniqueServiceName.SanitizeFileName());
|
|
WriteObject(filename, new MemoryStream(ssdpMetadataByteArray));
|
|
}
|
|
|
|
public byte[] SsdpGetMetadata(SsdpDevice ssdpDevice)
|
|
{
|
|
string filename = String.Format("/SSDP/{0}.xml", ssdpDevice.UniqueServiceName.SanitizeFileName());
|
|
byte[] bytes = GetObject(filename);
|
|
return bytes;
|
|
}
|
|
}
|
|
}
|