ft/skyscraper8
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Can now extract segments from Threemedia OTT.
All checks were successful
🚀 Pack skyscraper8 / make-zip (push) Successful in 1m57s
Details

Browse Source
This commit is contained in:
feyris-tan 2026-06-28 21:37:14 +02:00
parent 124bc001aa
commit a6f888f144
18 changed files with 714 additions and 277 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
skyscraper8/Ietf/FLUTE/FecHeader.cs
View File

@ -14,6 +14,7 @@ namespace skyscraper8.Ietf.FLUTE
switch (codepoint)
{
case 0:
case 1:
SourceBlockNumber = ms.ReadUInt16BE();
EncodingSymbolId = ms.ReadUInt16BE();
break;

2
skyscraper8/Properties/launchSettings.json
View File

@ -2,7 +2,7 @@
"profiles": {
"skyscraper8": {
"commandName": "Project",
"commandLineArgs": "\"Z:\\Persönliches\\Satellitescommunity\\Skyscraper Test Fixture\\117W_4050_OTT.ts\"",
"commandLineArgs": "\"Z:\\Persönliches\\Satellitescommunity\\Skyscraper Test Fixture\\brazilian-dvb-nip-000000.ts\"",
"remoteDebugEnabled": false
},
"Container (Dockerfile)": {

18
skyscraper8/Skyscraper/IO/StreamExtensions.cs
View File

@ -364,11 +364,26 @@ namespace skyscraper5.Skyscraper.IO
public static void DumpToFile(this Stream stream, string filename)
{
long? currentPosition = null;
int wishedForBlockSize = 65536;
if (stream.CanSeek)
{
currentPosition = stream.Position;
stream.Seek(0, SeekOrigin.Begin);
wishedForBlockSize = 4096;
}
FileStream fileStream = File.OpenWrite(filename);
stream.CopyTo(fileStream);
stream.CopyTo(fileStream, wishedForBlockSize);
fileStream.Flush(true);
fileStream.Close();
fileStream.Dispose();
if (stream.CanSeek)
{
stream.Seek(currentPosition.Value, SeekOrigin.Begin);
}
}
public static int TryReadExactly(this Stream stream, byte[] buffer, int offset, int count)
@ -432,5 +447,6 @@ namespace skyscraper5.Skyscraper.IO
throw new EndOfStreamException("failed to read mac address");
return new PhysicalAddress(buffer);
}
}
}

14
skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs
View File

@ -97,6 +97,8 @@ using Tsubasa.IO;
using Platform = skyscraper5.Dvb.SystemSoftwareUpdate.Model.Platform;
using RntParser = skyscraper5.Dvb.TvAnytime.RntParser;
using skyscraper8.Dvb.DataBroadcasting;
using skyscraper8.Skyscraper.Scraper.Storage.Tar;
using skyscraper8.ThreemediaOtt;
namespace skyscraper5.Skyscraper.Scraper
{
@ -107,7 +109,7 @@ namespace skyscraper5.Skyscraper.Scraper
IAutodetectionEventHandler, IRstEventHandler, IRntEventHandler, IMultiprotocolEncapsulationEventHandler, ObjectCarouselEventHandler, T2MIEventHandler,
IDisposable, IFrameGrabberEventHandler, IntEventHandler, IRctEventHandler, ISkyscraperContext, IDocsisEventHandler, AbertisDecoderEventHandler, Id3Handler,
InteractionChannelHandler, SgtEventHandler, IDvbNipEventHandler, UleEventHandler, OtvSsuHandler, NdsSsuHandler, ISubTsHandler, ILldpFrameHandler, SisHandler, IWneHandler,
IAtscPlpEventHandler, IAtsc3EventHandler
IAtscPlpEventHandler, IAtsc3EventHandler, IThreeMediaEventHandler
{
public const bool ALLOW_STREAM_TYPE_AUTODETECTION = true;
public const bool ALLOW_FFMPEG_FRAMEGRABBER = true;
@ -3659,5 +3661,15 @@ namespace skyscraper5.Skyscraper.Scraper
//MPD has no clear identifier.
//Metadata Envelope has no clear identifier
}
public void OnThreeMediaFileDelivery(MemoryStream entryStream, TarHeader tarHeader, IReadOnlyDictionary<string, string> streamFooterData,
ushort destinationPort)
{
if (!ObjectStorage.TestForThreemediaSegment(tarHeader))
{
LogEvent(SkyscraperContextEvent.ThreeMediaOttFileDelivery,String.Format("{0} (from {1})", tarHeader.Filename, streamFooterData["Content-Location"]));
ObjectStorage.StoreThreemediaSegment(tarHeader, entryStream);
}
}
}
}

3
skyscraper8/Skyscraper/Scraper/SkyscraperContextEvent.cs
View File

@ -108,6 +108,7 @@
Atsc3Detected,
Atsc3ServiceFound,
Atsc3Segment,
Atsc3Held
Atsc3Held,
ThreeMediaOttFileDelivery
}
}

24
skyscraper8/Skyscraper/Scraper/Storage/Filesystem/FilesystemStorage.cs
View File

@ -47,6 +47,7 @@ using System.Net.NetworkInformation;
using System.Text;
using System.Text.RegularExpressions;
using skyscraper8.Atsc.A331.Schema;
using skyscraper8.Skyscraper.Scraper.Storage.Tar;
using Platform = skyscraper5.Dvb.SystemSoftwareUpdate.Model.Platform;
namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
@ -1851,5 +1852,28 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
stream.Dispose();
}
public bool TestForThreemediaSegment(TarHeader tarHeader)
{
string outfilename = Path.Combine(rootDirectory.FullName, "ThreemediaOTT", tarHeader.Filename);
FileInfo fi = new FileInfo(outfilename);
return fi.Exists;
}
public void StoreThreemediaSegment(TarHeader tarHeader, MemoryStream stream)
{
string outfilename = Path.Combine(rootDirectory.FullName, "ThreemediaOTT", tarHeader.Filename);
FileInfo fi = new FileInfo(outfilename);
fi.Directory.EnsureExists();
if (stream.CanSeek)
stream.Position = 0;
FileStream fileStream = fi.OpenWrite();
stream.CopyTo(fileStream);
fileStream.Flush();
fileStream.Close();
stream.Dispose();
}
}
}

11
skyscraper8/Skyscraper/Scraper/Storage/NullObjectStorage.cs
View File

@ -11,6 +11,7 @@ using skyscraper8.Experimentals.NdsSsu;
using skyscraper8.Ietf.FLUTE;
using skyscraper8.SimpleServiceDiscoveryProtocol;
using skyscraper8.Skyscraper.Drawing;
using skyscraper8.Skyscraper.Scraper.Storage.Tar;
namespace skyscraper8.Skyscraper.Scraper.Storage
{
@ -189,5 +190,15 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
{
throw new NotImplementedException();
}
public bool TestForThreemediaSegment(TarHeader tarHeader)
{
throw new NotImplementedException();
}
public void StoreThreemediaSegment(TarHeader tarHeader, MemoryStream memoryStream)
{
throw new NotImplementedException();
}
}
}

3
skyscraper8/Skyscraper/Scraper/Storage/ObjectStorage.cs
View File

@ -13,6 +13,7 @@ using skyscraper8.Experimentals.NdsSsu;
using skyscraper8.Ietf.FLUTE;
using skyscraper8.SimpleServiceDiscoveryProtocol;
using skyscraper8.Skyscraper.Drawing;
using skyscraper8.Skyscraper.Scraper.Storage.Tar;
namespace skyscraper8.Skyscraper.Scraper.Storage
{
@ -50,5 +51,7 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
void StoreWneStory(uint sessionId, string filename, Stream value);
bool TestForAtsc3Segment(IPEndPoint destination, string outFileName);
void StoreAtsc3Segment(IPEndPoint destination, string outFileName, Stream stream);
bool TestForThreemediaSegment(TarHeader tarHeader);
void StoreThreemediaSegment(TarHeader tarHeader, MemoryStream memoryStream);
}
}

29
skyscraper8/Skyscraper/Scraper/Storage/Tar/TarHeader.cs
View File

@ -1,5 +1,7 @@
using System.Text;
using Newtonsoft.Json.Converters;
using skyscraper5.Skyscraper;
using skyscraper5.Skyscraper.IO;
namespace skyscraper8.Skyscraper.Scraper.Storage.Tar;
@ -16,6 +18,8 @@ public class TarHeader
OwnerUsername = "sophia";
OwnerGroupName = "sophia";
}
private TarHeader() {}
private string _filename;
public string Filename
@ -80,7 +84,29 @@ public class TarHeader
if (buffer.Length != 512)
throw new InvalidDataException("Invalid TAR header length");
throw new NotImplementedException();
MemoryStream ms = new MemoryStream(buffer, false);
TarHeader tarHeader = new TarHeader();
tarHeader.Filename = ms.ReadUTF8FixedLength(100).Trim('\0');
tarHeader.Permissions = new UnixFilePermissions(ms.ReadUTF8FixedLength(8));
tarHeader.Uid = (int)OctalToLong(ms.ReadUTF8FixedLength(8));
tarHeader.Gid = (int)OctalToLong(ms.ReadUTF8FixedLength(8));
tarHeader.Size = OctalToLong(ms.ReadUTF8FixedLength(12));
tarHeader.ModificationTime = OctalToLong(ms.ReadUTF8FixedLength(12)).AsUnixtime();
long checksum = OctalToLong(ms.ReadUTF8FixedLength(8));
char filetype = (char)ms.ReadUInt8();
if (buffer[257] == 'u' && buffer[258] == 's' && buffer[259] == 't' && buffer[260] == 'a' && buffer[261] == 'r')
{
ms.Position = 265;
tarHeader.OwnerUsername = ms.ReadAsciiNullTerminated();
ms.Position = 297;
tarHeader.OwnerGroupName = ms.ReadAsciiNullTerminated();
}
//TODO: actually verify the checksum!
return tarHeader;
}
public byte[] Serialize()
@ -146,6 +172,7 @@ public class TarHeader
private static long OctalToLong(string value)
{
value = value.Trim('\0');
long result = 0;
for (int i = 0; i < value.Length; ++i)

510
skyscraper8/Skyscraper/Scraper/Storage/Tar/TarObjectStorage.cs
View File

@ -1,241 +1,241 @@
using moe.yo3explorer.skyscraper8.DVBI.Model;
using skyscraper5.Dvb.DataBroadcasting.SkyscraperVfs;
using skyscraper5.Dvb.Descriptors;
using skyscraper5.Skyscraper;
using skyscraper8.DvbNip;
using skyscraper8.Experimentals.NdsSsu;
using skyscraper8.Ietf.FLUTE;
using skyscraper8.SimpleServiceDiscoveryProtocol;
using skyscraper8.Skyscraper.Drawing;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace skyscraper8.Skyscraper.Scraper.Storage.Tar
{
public class TarObjectStorage : ObjectStorage
{
public TarObjectStorage(FileInfo fi)
{
tarFileInfo = fi;
tarStream = fi.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read);
tarArchive = new TarArchive(tarStream);
}
private FileInfo tarFileInfo;
private Stream tarStream;
private TarArchive tarArchive;
public bool SsdpDeviceKnown(SsdpDevice ssdpDevice)
{
string filename = string.Format("ssdp/{0}.xml", ssdpDevice.UniqueServiceName.SanitizeFileName());
return tarArchive.HasEntry(filename);
}
public void SsdpStoreMetadata(SsdpDevice ssdpDevice, byte[] ssdpMetadataByteArray)
{
string filename = string.Format("ssdp/{0}.xml", ssdpDevice.UniqueServiceName.SanitizeFileName());
tarArchive.WriteEntry(filename, ssdpMetadataByteArray);
}
public byte[] SsdpGetMetadata(SsdpDevice ssdpDevice)
{
string filename = string.Format("ssdp/{0}.xml", ssdpDevice.UniqueServiceName.SanitizeFileName());
return tarArchive.ReadEntry(filename);
}
public bool ObjectCarouselFileArrival(VfsFile vfsFile, int transportStreamId, int networkId)
{
string filename = string.Format("dsm-cc/objects/{0}/{1}/{2}/{3}", networkId, transportStreamId, vfsFile.SourcePid, vfsFile.ToString());
if (tarArchive.HasEntry(filename))
{
return false;
}
else
{
tarArchive.WriteEntry(filename, vfsFile.FileContent);
return true;
}
}
public void DataCarouselModuleArrival(int currentNetworkId, int currentTransportStreamId, int elementaryPid,
ushort moduleModuleId, byte moduleModuleVersion, Stream result)
{
string filename = string.Format("dsm-ccc/{0}/{1}/{2}/{3}_V{4}.bin", currentNetworkId, currentTransportStreamId, elementaryPid, moduleModuleId, moduleModuleVersion);
tarArchive.WriteEntry(filename, result);
}
public bool IsDsmCcModuleWanted(int currentNetworkId, int currentTransportStreamId, int elementaryPid, ushort moduleId,
byte moduleVersion)
{
string filename = string.Format("dsm-ccc/{0}/{1}/{2}/{3}_V{4}.bin", currentNetworkId, currentTransportStreamId, elementaryPid, moduleId, moduleVersion);
return !tarArchive.HasEntry(filename);
}
public bool TestForFramegrab(int currentNetworkId, int transportStreamId, ushort mappingProgramNumber,
int mappingStreamElementaryPid)
{
string filename = string.Format("screenshots/master/{0}/{1}/{2}_{3}.jpg", currentNetworkId, transportStreamId, mappingProgramNumber, mappingStreamElementaryPid);
return tarArchive.HasEntry(filename);
}
public void StoreFramegrab(int currentNetworkId, int transportStreamId, ushort mappingProgramNumber, ushort pid,
byte[] imageData)
{
string filename = string.Format("screenshots/master/{0}/{1}/{2}_{3}.jpg", currentNetworkId, transportStreamId, mappingProgramNumber, pid);
tarArchive.WriteEntry(filename, imageData);
}
public void WaitForCompletion()
{
}
public void UiSetVersion(int version)
{
throw new NotImplementedException();
}
public object[] GetPluginConnector()
{
return new object[] { tarArchive };
}
public void Ping()
{
byte[] buffer = new byte[4096];
for (int i = 0; i < buffer.Length; i++)
{
buffer[i] = (byte)i;
}
string filename = string.Format("test_write1.dvr");
if (!tarArchive.HasEntry(filename))
{
tarArchive.WriteEntry(filename, buffer);
}
byte[] cmpBuffer = tarArchive.ReadEntry(filename);
if (buffer.Length != cmpBuffer.Length)
{
throw new Exception("readback produced invalid length");
}
for (int i = 0; i < cmpBuffer.Length; i++)
{
if (buffer[i] != cmpBuffer[i])
{
throw new Exception("readback produced unexpected result");
}
}
}
public bool DvbNipTestForFile(string announcedFileContentLocation)
{
string filename = "/nip/" + DvbNipUtilities.MakeFilename(announcedFileContentLocation);
return tarArchive.HasEntry(filename);
}
public void DvbNipFileArrival(NipActualCarrierInformation carrier, FluteListener listener)
{
string filename = "/nip/" + DvbNipUtilities.MakeFilename(listener.FileAssociation.ContentLocation);
tarArchive.WriteEntry(filename, listener.ToStream());
}
public void StoreIqGraph(Guid jobGuid, long frequency, char polarity, IqChartData plot)
{
string filename = String.Format("scandata/{0}/{1}_{2}.iq", jobGuid, frequency, polarity);
byte[] plotBuffer = plot.SaveToBytes();
tarArchive.WriteEntry(filename, plotBuffer);
}
public void StoreRfSpectrum(Guid jobGuid, RfSpectrumData rfSpectrum)
{
string filename = String.Format("scandata/{0}/index.rf", jobGuid);
byte[] plotBuffer = rfSpectrum.SaveToBytes();
tarArchive.WriteEntry(filename, plotBuffer);
}
public void DeleteIqGraph(Guid jobGuid, int frequency, SatelliteDeliverySystemDescriptor.PolarizationEnum frequencyItem2)
{
char polarity = frequencyItem2.ToString()[0];
string filename = String.Format("scandata/{0}/{1}_{2}.iq.deleted", jobGuid, frequency, polarity);
tarArchive.WriteEntry(filename, new byte[1]);
}
public void DeleteRfSpectrum(Guid selectedGuid)
{
string filename = String.Format("scandata/{0}/index.rf.deleted", selectedGuid);
tarArchive.WriteEntry(filename, new byte[1]);
}
public bool OtvSsuTestFile(int? currentNetworkId, int? currentTransportStreamId, int sourcePid, ushort tableIdExtension,
uint fileId, uint unknown1, uint length)
{
string filename = String.Format("otv-ssu/{0}/{1}/{2}/{3}.bin", currentNetworkId, currentTransportStreamId, sourcePid, tableIdExtension);
return tarArchive.HasEntry(filename);
}
public void OnOtvSsuComplete(int? currentNetworkId, int? currentTransportStreamId, int sourcePid, Stream getStream,
ushort tableIdExtension, uint fileId, uint unknown1, uint length)
{
string filename = String.Format("otv-ssu/{0}/{1}/{2}/{3}.bin", currentNetworkId, currentTransportStreamId, sourcePid, tableIdExtension);
tarArchive.WriteEntry(filename, getStream);
}
public void OnNdsSsuComplete(int? currentNetworkId, int? currentTransportStreamId, int pid, ushort tableIdExtension,
NdsSsuDataMap dataMap)
{
string filename = String.Format("nds-ssu/{0}/{1}/{2}/{3}.bin", currentNetworkId, currentTransportStreamId, pid, tableIdExtension);
tarArchive.WriteEntry(filename, dataMap.WriteToByteArray());
}
public bool NdsSsuTestFile(int? currentNetworkId, int? currentTransportStreamId, int pid, ushort tableIdExtension)
{
string filename = String.Format("nds-ssu/{0}/{1}/{2}/{3}.bin", currentNetworkId, currentTransportStreamId, pid, tableIdExtension);
return tarArchive.HasEntry(filename);
}
public bool TestForSisDsaci(int nid, int tsid, ushort groupId, int versionNumber)
{
string filename = String.Format("dvb-sis/{0}/{1}/Group{2}_Version{3}.xml", nid, tsid, groupId, versionNumber);
return tarArchive.HasEntry(filename);
}
public void StoreSisDsaci(int nid, int tsid, ushort currentDsaGroupId, int versionNumber, Stream dsaci)
{
string filename = String.Format("dvb-sis/{0}/{1}/Group{2}_Version{3}.xml", nid, tsid, currentDsaGroupId, versionNumber);
tarArchive.WriteEntry(filename, dsaci);
}
public bool TestForObjectCarouselFileArrival(string vfsFile, int transportStreamId, int pid, int networkId)
{
string filename = string.Format("dsm-cc/objects/{0}/{1}/{2}/{3}", networkId, transportStreamId, pid, vfsFile.ToString());
return tarArchive.HasEntry(filename);
}
public byte[] GetObjectCarouselFileArrival(string vfsFile, int transportStreamId, int pid, int networkId)
{
string filename = string.Format("dsm-cc/objects/{0}/{1}/{2}/{3}", networkId, transportStreamId, pid, vfsFile.ToString());
return tarArchive.ReadEntry(filename);
}
public byte[] DvbNipGetFile(string path)
{
string filename = "/nip/" + DvbNipUtilities.MakeFilename(path);
return tarArchive.ReadEntry(filename);
}
public RfSpectrumData GetRfSpectrum(Guid selectedGuid)
{
string filename = String.Format("scandata/{0}/index.rf", selectedGuid);
byte[] buffer = tarArchive.ReadEntry(filename);
RfSpectrumData rfSpectrumData = RfSpectrumData.LoadFromStream(new MemoryStream(buffer));
return rfSpectrumData;
using moe.yo3explorer.skyscraper8.DVBI.Model;
using skyscraper5.Dvb.DataBroadcasting.SkyscraperVfs;
using skyscraper5.Dvb.Descriptors;
using skyscraper5.Skyscraper;
using skyscraper8.DvbNip;
using skyscraper8.Experimentals.NdsSsu;
using skyscraper8.Ietf.FLUTE;
using skyscraper8.SimpleServiceDiscoveryProtocol;
using skyscraper8.Skyscraper.Drawing;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace skyscraper8.Skyscraper.Scraper.Storage.Tar
{
public class TarObjectStorage : ObjectStorage
{
public TarObjectStorage(FileInfo fi)
{
tarFileInfo = fi;
tarStream = fi.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read);
tarArchive = new TarArchive(tarStream);
}
private FileInfo tarFileInfo;
private Stream tarStream;
private TarArchive tarArchive;
public bool SsdpDeviceKnown(SsdpDevice ssdpDevice)
{
string filename = string.Format("ssdp/{0}.xml", ssdpDevice.UniqueServiceName.SanitizeFileName());
return tarArchive.HasEntry(filename);
}
public void SsdpStoreMetadata(SsdpDevice ssdpDevice, byte[] ssdpMetadataByteArray)
{
string filename = string.Format("ssdp/{0}.xml", ssdpDevice.UniqueServiceName.SanitizeFileName());
tarArchive.WriteEntry(filename, ssdpMetadataByteArray);
}
public byte[] SsdpGetMetadata(SsdpDevice ssdpDevice)
{
string filename = string.Format("ssdp/{0}.xml", ssdpDevice.UniqueServiceName.SanitizeFileName());
return tarArchive.ReadEntry(filename);
}
public bool ObjectCarouselFileArrival(VfsFile vfsFile, int transportStreamId, int networkId)
{
string filename = string.Format("dsm-cc/objects/{0}/{1}/{2}/{3}", networkId, transportStreamId, vfsFile.SourcePid, vfsFile.ToString());
if (tarArchive.HasEntry(filename))
{
return false;
}
else
{
tarArchive.WriteEntry(filename, vfsFile.FileContent);
return true;
}
}
public void DataCarouselModuleArrival(int currentNetworkId, int currentTransportStreamId, int elementaryPid,
ushort moduleModuleId, byte moduleModuleVersion, Stream result)
{
string filename = string.Format("dsm-ccc/{0}/{1}/{2}/{3}_V{4}.bin", currentNetworkId, currentTransportStreamId, elementaryPid, moduleModuleId, moduleModuleVersion);
tarArchive.WriteEntry(filename, result);
}
public bool IsDsmCcModuleWanted(int currentNetworkId, int currentTransportStreamId, int elementaryPid, ushort moduleId,
byte moduleVersion)
{
string filename = string.Format("dsm-ccc/{0}/{1}/{2}/{3}_V{4}.bin", currentNetworkId, currentTransportStreamId, elementaryPid, moduleId, moduleVersion);
return !tarArchive.HasEntry(filename);
}
public bool TestForFramegrab(int currentNetworkId, int transportStreamId, ushort mappingProgramNumber,
int mappingStreamElementaryPid)
{
string filename = string.Format("screenshots/master/{0}/{1}/{2}_{3}.jpg", currentNetworkId, transportStreamId, mappingProgramNumber, mappingStreamElementaryPid);
return tarArchive.HasEntry(filename);
}
public void StoreFramegrab(int currentNetworkId, int transportStreamId, ushort mappingProgramNumber, ushort pid,
byte[] imageData)
{
string filename = string.Format("screenshots/master/{0}/{1}/{2}_{3}.jpg", currentNetworkId, transportStreamId, mappingProgramNumber, pid);
tarArchive.WriteEntry(filename, imageData);
}
public void WaitForCompletion()
{
}
public void UiSetVersion(int version)
{
throw new NotImplementedException();
}
public object[] GetPluginConnector()
{
return new object[] { tarArchive };
}
public void Ping()
{
byte[] buffer = new byte[4096];
for (int i = 0; i < buffer.Length; i++)
{
buffer[i] = (byte)i;
}
string filename = string.Format("test_write1.dvr");
if (!tarArchive.HasEntry(filename))
{
tarArchive.WriteEntry(filename, buffer);
}
byte[] cmpBuffer = tarArchive.ReadEntry(filename);
if (buffer.Length != cmpBuffer.Length)
{
throw new Exception("readback produced invalid length");
}
for (int i = 0; i < cmpBuffer.Length; i++)
{
if (buffer[i] != cmpBuffer[i])
{
throw new Exception("readback produced unexpected result");
}
}
}
public bool DvbNipTestForFile(string announcedFileContentLocation)
{
string filename = "/nip/" + DvbNipUtilities.MakeFilename(announcedFileContentLocation);
return tarArchive.HasEntry(filename);
}
public void DvbNipFileArrival(NipActualCarrierInformation carrier, FluteListener listener)
{
string filename = "/nip/" + DvbNipUtilities.MakeFilename(listener.FileAssociation.ContentLocation);
tarArchive.WriteEntry(filename, listener.ToStream());
}
public void StoreIqGraph(Guid jobGuid, long frequency, char polarity, IqChartData plot)
{
string filename = String.Format("scandata/{0}/{1}_{2}.iq", jobGuid, frequency, polarity);
byte[] plotBuffer = plot.SaveToBytes();
tarArchive.WriteEntry(filename, plotBuffer);
}
public void StoreRfSpectrum(Guid jobGuid, RfSpectrumData rfSpectrum)
{
string filename = String.Format("scandata/{0}/index.rf", jobGuid);
byte[] plotBuffer = rfSpectrum.SaveToBytes();
tarArchive.WriteEntry(filename, plotBuffer);
}
public void DeleteIqGraph(Guid jobGuid, int frequency, SatelliteDeliverySystemDescriptor.PolarizationEnum frequencyItem2)
{
char polarity = frequencyItem2.ToString()[0];
string filename = String.Format("scandata/{0}/{1}_{2}.iq.deleted", jobGuid, frequency, polarity);
tarArchive.WriteEntry(filename, new byte[1]);
}
public void DeleteRfSpectrum(Guid selectedGuid)
{
string filename = String.Format("scandata/{0}/index.rf.deleted", selectedGuid);
tarArchive.WriteEntry(filename, new byte[1]);
}
public bool OtvSsuTestFile(int? currentNetworkId, int? currentTransportStreamId, int sourcePid, ushort tableIdExtension,
uint fileId, uint unknown1, uint length)
{
string filename = String.Format("otv-ssu/{0}/{1}/{2}/{3}.bin", currentNetworkId, currentTransportStreamId, sourcePid, tableIdExtension);
return tarArchive.HasEntry(filename);
}
public void OnOtvSsuComplete(int? currentNetworkId, int? currentTransportStreamId, int sourcePid, Stream getStream,
ushort tableIdExtension, uint fileId, uint unknown1, uint length)
{
string filename = String.Format("otv-ssu/{0}/{1}/{2}/{3}.bin", currentNetworkId, currentTransportStreamId, sourcePid, tableIdExtension);
tarArchive.WriteEntry(filename, getStream);
}
public void OnNdsSsuComplete(int? currentNetworkId, int? currentTransportStreamId, int pid, ushort tableIdExtension,
NdsSsuDataMap dataMap)
{
string filename = String.Format("nds-ssu/{0}/{1}/{2}/{3}.bin", currentNetworkId, currentTransportStreamId, pid, tableIdExtension);
tarArchive.WriteEntry(filename, dataMap.WriteToByteArray());
}
public bool NdsSsuTestFile(int? currentNetworkId, int? currentTransportStreamId, int pid, ushort tableIdExtension)
{
string filename = String.Format("nds-ssu/{0}/{1}/{2}/{3}.bin", currentNetworkId, currentTransportStreamId, pid, tableIdExtension);
return tarArchive.HasEntry(filename);
}
public bool TestForSisDsaci(int nid, int tsid, ushort groupId, int versionNumber)
{
string filename = String.Format("dvb-sis/{0}/{1}/Group{2}_Version{3}.xml", nid, tsid, groupId, versionNumber);
return tarArchive.HasEntry(filename);
}
public void StoreSisDsaci(int nid, int tsid, ushort currentDsaGroupId, int versionNumber, Stream dsaci)
{
string filename = String.Format("dvb-sis/{0}/{1}/Group{2}_Version{3}.xml", nid, tsid, currentDsaGroupId, versionNumber);
tarArchive.WriteEntry(filename, dsaci);
}
public bool TestForObjectCarouselFileArrival(string vfsFile, int transportStreamId, int pid, int networkId)
{
string filename = string.Format("dsm-cc/objects/{0}/{1}/{2}/{3}", networkId, transportStreamId, pid, vfsFile.ToString());
return tarArchive.HasEntry(filename);
}
public byte[] GetObjectCarouselFileArrival(string vfsFile, int transportStreamId, int pid, int networkId)
{
string filename = string.Format("dsm-cc/objects/{0}/{1}/{2}/{3}", networkId, transportStreamId, pid, vfsFile.ToString());
return tarArchive.ReadEntry(filename);
}
public byte[] DvbNipGetFile(string path)
{
string filename = "/nip/" + DvbNipUtilities.MakeFilename(path);
return tarArchive.ReadEntry(filename);
}
public RfSpectrumData GetRfSpectrum(Guid selectedGuid)
{
string filename = String.Format("scandata/{0}/index.rf", selectedGuid);
byte[] buffer = tarArchive.ReadEntry(filename);
RfSpectrumData rfSpectrumData = RfSpectrumData.LoadFromStream(new MemoryStream(buffer));
return rfSpectrumData;
}
public bool TestForWneStory(uint sessionId, string filename)
@ -247,15 +247,25 @@ namespace skyscraper8.Skyscraper.Scraper.Storage.Tar
{
throw new NotImplementedException();
}
public bool TestForAtsc3Segment(IPEndPoint destination, string outFileName)
{
throw new NotImplementedException();
}
public void StoreAtsc3Segment(IPEndPoint destination, string outFileName, Stream stream)
{
throw new NotImplementedException();
}
}
}
public bool TestForAtsc3Segment(IPEndPoint destination, string outFileName)
{
throw new NotImplementedException();
}
public void StoreAtsc3Segment(IPEndPoint destination, string outFileName, Stream stream)
{
throw new NotImplementedException();
}
public bool TestForThreemediaSegment(TarHeader tarHeader)
{
throw new NotImplementedException();
}
public void StoreThreemediaSegment(TarHeader tarHeader, MemoryStream memoryStream)
{
throw new NotImplementedException();
}
}
}

4
skyscraper8/Skyscraper/Scraper/Storage/Tar/UnixFilePermissions.cs
View File

@ -12,6 +12,10 @@ public class UnixFilePermissions
public UnixFilePermissions(string permissionString)
{
if (permissionString.Length == 8)
{
permissionString = permissionString.Substring(0, 7);
}
if (permissionString.Length == 7)
permissionString = permissionString.Substring(4);

14
skyscraper8/ThreemediaOtt/IThreeMediaEventHandler.cs Normal file
View File

@ -0,0 +1,14 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using skyscraper8.Skyscraper.Scraper.Storage.Tar;
namespace skyscraper8.ThreemediaOtt
{
internal interface IThreeMediaEventHandler
{
void OnThreeMediaFileDelivery(MemoryStream entryStream, TarHeader tarHeader, IReadOnlyDictionary<string, string> streamFooterData, ushort destinationPort);
}
}

2
skyscraper8/ThreemediaOtt/ThreemediaBaseStream.cs
View File

@ -53,6 +53,8 @@ namespace skyscraper8.ThreemediaOtt
throw new IOException("An attempt was made to move the file pointer before the beginning or past the end of the stream.");
}
_internalPosition = newPosition;
SeekHook();
return newPosition;

57
skyscraper8/ThreemediaOtt/ThreemediaContentFragment.cs
View File

@ -1,15 +1,17 @@
using System;
using log4net;
using skyscraper5.Skyscraper.IO;
using skyscraper8.Ietf.FLUTE;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using skyscraper5.Skyscraper.IO;
using skyscraper8.Ietf.FLUTE;
namespace skyscraper8.ThreemediaOtt
{
internal class ThreemediaContentFragment : IDisposable
{
private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
public void IngestPacket(LctFrame lctFrame)
{
int totalBlocks = lctFrame.FecHeader.SourceBlockLength.Value;
@ -63,14 +65,53 @@ namespace skyscraper8.ThreemediaOtt
int footerLength = (int)lastBlockStream.ReadUInt32BE();
int footerOffset = (int)(lastBlockStream.Length - 8 - footerLength);
Dictionary<string, string> footer = ExtractFooter(lastBlock, footerLength, footerOffset);
if (footerOffset < 0)
{
logger.Info("Detected ThreeMedia Content Fragment with oversized buffer.");
//negative footer offset, will need to assemble two blocks.
byte[] lastBlockUpper = blocks[blocks.Length - 2];
byte[] lastBlockLower = blocks[blocks.Length - 1];
lastBlock = new byte[lastBlockUpper.Length + lastBlockLower.Length];
Array.Copy(lastBlockUpper,0,lastBlock,0,lastBlockUpper.Length);
Array.Copy(lastBlockLower, 0, lastBlock, lastBlockUpper.Length, lastBlockLower.Length);
footerOffset = (int)(lastBlock.Length - 8 - footerLength);
Dictionary<string, string> footer = ExtractFooter(lastBlock, footerLength, footerOffset);
int[] blockSizes = Array.ConvertAll(blocks, (x) => x.Length);
blockSizes[blocks.Length - 1] = footerOffset;
int[] blockSizes = Array.ConvertAll(blocks, (x) => x.Length);
AdjustBlockSizeForFooter(blockSizes, 8 + footerLength);
ThreemediaContentFragmentStream result = new ThreemediaContentFragmentStream(blocks, blockSizes, footer);
ThreemediaContentFragmentStream result = new ThreemediaContentFragmentStream(blocks, blockSizes, footer);
return result;
}
else
{
//Footer is complete in one block.
Dictionary<string, string> footer = ExtractFooter(lastBlock, footerLength, footerOffset);
throw new NotImplementedException();
int[] blockSizes = Array.ConvertAll(blocks, (x) => x.Length);
blockSizes[blocks.Length - 1] = footerOffset;
//AdjustBlockSizeForFooter(blockSizes, footerOffset);
ThreemediaContentFragmentStream result = new ThreemediaContentFragmentStream(blocks, blockSizes, footer);
return result;
}
}
private void AdjustBlockSizeForFooter(int[] blockSizes, int footerOffset)
{
int currentItem = blockSizes.Length - 1;
while (footerOffset > 0)
{
int edible = Math.Min(footerOffset, blockSizes[currentItem]);
footerOffset -= edible;
blockSizes[currentItem] -= edible;
if (footerOffset > 0)
{
currentItem--;
}
}
}
private Dictionary<string, string> ExtractFooter(byte[] lastBlock, int footerLength, int footerOffset)

55
skyscraper8/ThreemediaOtt/ThreemediaContentFragmentStream.cs
View File

@ -10,7 +10,8 @@ namespace skyscraper8.ThreemediaOtt
{
private readonly byte[][] _blocks;
private readonly int[] _blockSizes;
private IReadOnlyDictionary<string, string> footer;
private int _currentBlockIndex;
private int _currentBlockPosition;
@ -26,11 +27,53 @@ namespace skyscraper8.ThreemediaOtt
// Calculate total stream length based on the valid payloads
SetLength(_blockSizes.Sum(x => (long)x));
this.footer = footer;
}
public override int Read(byte[] buffer, int offset, int count)
{
throw new NotImplementedException();
if (buffer == null)
throw new ArgumentNullException(nameof(buffer));
if (offset < 0 || count < 0 || offset + count > buffer.Length)
throw new ArgumentException("Invalid buffer offset/count.");
if (_internalPosition >= Length || count == 0)
return 0;
int totalBytesRead = 0;
while (count > 0 && _currentBlockIndex < _blocks.Length)
{
int remainingInBlock = _blockSizes[_currentBlockIndex] - _currentBlockPosition;
// If the current block is fully read, move to the next one
if (remainingInBlock <= 0)
{
_currentBlockIndex++;
_currentBlockPosition = 0;
continue;
}
// Determine how much we can actually copy from the current block
int bytesToCopy = Math.Min(count, remainingInBlock);
Buffer.BlockCopy(
src: _blocks[_currentBlockIndex],
srcOffset: _currentBlockPosition,
dst: buffer,
dstOffset: offset + totalBytesRead,
count: bytesToCopy
);
// Update state
_currentBlockPosition += bytesToCopy;
_internalPosition += bytesToCopy;
totalBytesRead += bytesToCopy;
count -= bytesToCopy;
}
return totalBytesRead;
}
protected override void SeekHook()
@ -54,5 +97,13 @@ namespace skyscraper8.ThreemediaOtt
}
}
}
public IReadOnlyDictionary<string, string> FooterData
{
get
{
return footer;
}
}
}
}

59
skyscraper8/ThreemediaOtt/ThreemediaOttHandler.cs
View File

@ -4,12 +4,15 @@ using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using Ionic.Zlib;
using log4net;
using skyscraper5.Ietf.Rfc768;
using skyscraper5.Ietf.Rfc971;
using skyscraper5.Skyscraper.IO;
using skyscraper5.Skyscraper.Plugins;
using skyscraper5.Skyscraper.Scraper;
using skyscraper8.Ietf.FLUTE;
using skyscraper8.Skyscraper.Scraper.Storage.Tar;
namespace skyscraper8.ThreemediaOtt
{
@ -24,12 +27,12 @@ namespace skyscraper8.ThreemediaOtt
throw new NotImplementedException();
}
private SkyscraperContext context;
private IThreeMediaEventHandler eventHandler;
public void SetContext(DateTime? currentTime, object skyscraperContext)
{
if (context != null)
if (eventHandler == null)
{
context = skyscraperContext as SkyscraperContext;
eventHandler = skyscraperContext as IThreeMediaEventHandler;
}
}
@ -104,15 +107,55 @@ namespace skyscraper8.ThreemediaOtt
{
logger.WarnFormat(String.Format("The session on Port {0} is incomplete and can not be recovered.", destinationPort));
selectedSession.Dispose();
sessions.Remove(destinationPort);
return;
}
ThreemediaSessionStream stream = selectedSession.CreateStream();
string filetype = Path.GetExtension(stream.ContentLocation);
switch (filetype)
{
case ".tgz":
ExtractTarGz(stream,destinationPort);
stream.Dispose();
selectedSession.Dispose();
sessions.Remove(destinationPort);
return;
default:
logger.WarnFormat(String.Format("Unknown file type in ThreeMedia OTT Session: {0}", stream.ContentLocation));
break;
}
}
FileStream fs = File.OpenWrite("test.bin");
stream.CopyTo(fs);
fs.Flush(true);
fs.Close();
private void ExtractTarGz(ThreemediaSessionStream stream, ushort destinationPort)
{
GZipStream gz = new GZipStream(stream, CompressionMode.Decompress, false);
byte[] headerBuffer = new byte[512];
while (true)
{
int headerBufferResult = gz.Read(headerBuffer, 0, headerBuffer.Length);
if (headerBuffer[0] == 0)
break;
if (headerBufferResult == 0)
break;
TarHeader tarHeader = TarHeader.Deserialize(headerBuffer);
if (tarHeader == null)
break;
long blockyFileSize = tarHeader.Size;
bool needExtraBlock = blockyFileSize % 512 != 0;
blockyFileSize /= 512;
if (needExtraBlock)
{
blockyFileSize++;
}
blockyFileSize *= 512;
byte[] entryBuffer = gz.ReadBytes(blockyFileSize);
MemoryStream entryStream = new MemoryStream(entryBuffer, 0, (int)blockyFileSize, false);
eventHandler.OnThreeMediaFileDelivery(entryStream, tarHeader, stream.FooterData, destinationPort);
}
}
public bool StopProcessingAfterThis()

7
skyscraper8/ThreemediaOtt/ThreemediaSession.cs
View File

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using skyscraper5.Skyscraper.IO;
using skyscraper8.Ietf.FLUTE;
namespace skyscraper8.ThreemediaOtt
@ -17,6 +18,12 @@ namespace skyscraper8.ThreemediaOtt
int currentFragment = (int)lctFrame.LctHeader.TransportObjectIdentifier;
currentFragment--;
int fragmentsBound = fragments.GetUpperBound(0);
if (currentFragment > fragmentsBound)
{
return;
}
if (fragments[currentFragment] == null)
fragments[currentFragment] = new ThreemediaContentFragment();

178
skyscraper8/ThreemediaOtt/ThreemediaSessionStream.cs
View File

@ -8,19 +8,189 @@ namespace skyscraper8.ThreemediaOtt
{
internal class ThreemediaSessionStream : ThreemediaBaseStream
{
public ThreemediaSessionStream(ThreemediaContentFragmentStream[] fragmentStreams)
private readonly ThreemediaContentFragmentStream[] _fragments;
private int _currentFragmentIndex;
private IReadOnlyDictionary<string, string> footerData;
public ThreemediaSessionStream(ThreemediaContentFragmentStream[] fragments)
{
throw new NotImplementedException();
_fragments = fragments ?? throw new ArgumentNullException(nameof(fragments));
// Ensure none of the passed streams are null
if (_fragments.Any(f => f == null))
{
throw new ArgumentException("Fragment array cannot contain null elements.", nameof(fragments));
}
// Calculate the total combined length of all fragments
SetLength(_fragments.Sum(f => f.Length));
// Ensure all fragments start at position 0
ResetFragmentPositions();
ParseFooterData();
}
private void ParseFooterData()
{
Dictionary<string, string> result = new Dictionary<string, string>();
foreach (ThreemediaContentFragmentStream fragment in _fragments)
{
IReadOnlyDictionary<string, string> dictionary = fragment.FooterData;
foreach (KeyValuePair<string,string> key in dictionary)
{
result[key.Key] = key.Value;
}
}
footerData = result;
}
private void ResetFragmentPositions()
{
foreach (var fragment in _fragments)
{
fragment.Position = 0;
}
_currentFragmentIndex = 0;
_internalPosition = 0;
}
public override int Read(byte[] buffer, int offset, int count)
{
throw new NotImplementedException();
if (buffer == null)
throw new ArgumentNullException(nameof(buffer));
if (offset < 0 || count < 0 || offset + count > buffer.Length)
throw new ArgumentException("Invalid buffer offset/count.");
if (_internalPosition >= Length || count == 0)
return 0;
int totalBytesRead = 0;
while (count > 0 && _currentFragmentIndex < _fragments.Length)
{
ThreemediaContentFragmentStream currentFragment = _fragments[_currentFragmentIndex];
// Read from the current active fragment stream
int bytesRead = currentFragment.Read(buffer, offset + totalBytesRead, count);
if (bytesRead == 0)
{
// Current fragment is exhausted, advance to the next one
_currentFragmentIndex++;
continue;
}
totalBytesRead += bytesRead;
_internalPosition += bytesRead;
count -= bytesRead;
}
return totalBytesRead;
}
protected override void SeekHook()
{
throw new NotImplementedException();
long remainingOffset = _internalPosition;
_currentFragmentIndex = 0;
for (int i = 0; i < _fragments.Length; i++)
{
ThreemediaContentFragmentStream fragment = _fragments[i];
if (remainingOffset >= fragment.Length)
{
// This fragment is entirely behind our new position.
// Set its pointer to its end so it's ready if we backtrack later.
fragment.Position = fragment.Length;
remainingOffset -= fragment.Length;
_currentFragmentIndex++;
}
else
{
// Our target position lands right inside this fragment.
fragment.Position = remainingOffset;
_currentFragmentIndex = i;
remainingOffset = 0;
// Reset any subsequent fragments to 0 just in case we skipped forward past them earlier
for (int j = i + 1; j < _fragments.Length; j++)
{
_fragments[j].Position = 0;
}
break;
}
}
// Edge case: if we seeked exactly to the end of the entire stream
if (_currentFragmentIndex >= _fragments.Length)
{
_currentFragmentIndex = _fragments.Length;
}
}
public string ContentBase
{
get
{
return footerData?["Content-Base"];
}
}
public string ContentLocation
{
get
{
return footerData?["Content-Location"];
}
}
public int ContentFilesize
{
get
{
return Convert.ToInt32(footerData?["Content-Filesize"]);
}
}
public int ContentLength
{
get
{
return Convert.ToInt32(footerData?["Content-Length"]);
}
}
public string ContentFragment
{
get
{
return footerData?["Content-Fragment"];
}
}
public int ContentOffset
{
get
{
return Convert.ToInt32(footerData?["Content-Offset"]);
}
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
// Clean up all underlying fragment streams automatically
foreach (var fragment in _fragments)
{
fragment.Dispose();
}
}
base.Dispose(disposing);
}
public IReadOnlyDictionary<string, string> FooterData => footerData;
}
}