Can now extract segments from Threemedia OTT.
All checks were successful
🚀 Pack skyscraper8 / make-zip (push) Successful in 1m57s

This commit is contained in:
feyris-tan 2026-06-28 21:37:14 +02:00
parent 124bc001aa
commit a6f888f144
18 changed files with 714 additions and 277 deletions

View File

@ -14,6 +14,7 @@ namespace skyscraper8.Ietf.FLUTE
switch (codepoint)
{
case 0:
case 1:
SourceBlockNumber = ms.ReadUInt16BE();
EncodingSymbolId = ms.ReadUInt16BE();
break;

View File

@ -2,7 +2,7 @@
"profiles": {
"skyscraper8": {
"commandName": "Project",
"commandLineArgs": "\"Z:\\Persönliches\\Satellitescommunity\\Skyscraper Test Fixture\\117W_4050_OTT.ts\"",
"commandLineArgs": "\"Z:\\Persönliches\\Satellitescommunity\\Skyscraper Test Fixture\\brazilian-dvb-nip-000000.ts\"",
"remoteDebugEnabled": false
},
"Container (Dockerfile)": {

View File

@ -364,11 +364,26 @@ namespace skyscraper5.Skyscraper.IO
public static void DumpToFile(this Stream stream, string filename)
{
long? currentPosition = null;
int wishedForBlockSize = 65536;
if (stream.CanSeek)
{
currentPosition = stream.Position;
stream.Seek(0, SeekOrigin.Begin);
wishedForBlockSize = 4096;
}
FileStream fileStream = File.OpenWrite(filename);
stream.CopyTo(fileStream);
stream.CopyTo(fileStream, wishedForBlockSize);
fileStream.Flush(true);
fileStream.Close();
fileStream.Dispose();
if (stream.CanSeek)
{
stream.Seek(currentPosition.Value, SeekOrigin.Begin);
}
}
public static int TryReadExactly(this Stream stream, byte[] buffer, int offset, int count)
@ -432,5 +447,6 @@ namespace skyscraper5.Skyscraper.IO
throw new EndOfStreamException("failed to read mac address");
return new PhysicalAddress(buffer);
}
}
}

View File

@ -97,6 +97,8 @@ using Tsubasa.IO;
using Platform = skyscraper5.Dvb.SystemSoftwareUpdate.Model.Platform;
using RntParser = skyscraper5.Dvb.TvAnytime.RntParser;
using skyscraper8.Dvb.DataBroadcasting;
using skyscraper8.Skyscraper.Scraper.Storage.Tar;
using skyscraper8.ThreemediaOtt;
namespace skyscraper5.Skyscraper.Scraper
{
@ -107,7 +109,7 @@ namespace skyscraper5.Skyscraper.Scraper
IAutodetectionEventHandler, IRstEventHandler, IRntEventHandler, IMultiprotocolEncapsulationEventHandler, ObjectCarouselEventHandler, T2MIEventHandler,
IDisposable, IFrameGrabberEventHandler, IntEventHandler, IRctEventHandler, ISkyscraperContext, IDocsisEventHandler, AbertisDecoderEventHandler, Id3Handler,
InteractionChannelHandler, SgtEventHandler, IDvbNipEventHandler, UleEventHandler, OtvSsuHandler, NdsSsuHandler, ISubTsHandler, ILldpFrameHandler, SisHandler, IWneHandler,
IAtscPlpEventHandler, IAtsc3EventHandler
IAtscPlpEventHandler, IAtsc3EventHandler, IThreeMediaEventHandler
{
public const bool ALLOW_STREAM_TYPE_AUTODETECTION = true;
public const bool ALLOW_FFMPEG_FRAMEGRABBER = true;
@ -3659,5 +3661,15 @@ namespace skyscraper5.Skyscraper.Scraper
//MPD has no clear identifier.
//Metadata Envelope has no clear identifier
}
public void OnThreeMediaFileDelivery(MemoryStream entryStream, TarHeader tarHeader, IReadOnlyDictionary<string, string> streamFooterData,
ushort destinationPort)
{
if (!ObjectStorage.TestForThreemediaSegment(tarHeader))
{
LogEvent(SkyscraperContextEvent.ThreeMediaOttFileDelivery,String.Format("{0} (from {1})", tarHeader.Filename, streamFooterData["Content-Location"]));
ObjectStorage.StoreThreemediaSegment(tarHeader, entryStream);
}
}
}
}

View File

@ -108,6 +108,7 @@
Atsc3Detected,
Atsc3ServiceFound,
Atsc3Segment,
Atsc3Held
Atsc3Held,
ThreeMediaOttFileDelivery
}
}

View File

@ -47,6 +47,7 @@ using System.Net.NetworkInformation;
using System.Text;
using System.Text.RegularExpressions;
using skyscraper8.Atsc.A331.Schema;
using skyscraper8.Skyscraper.Scraper.Storage.Tar;
using Platform = skyscraper5.Dvb.SystemSoftwareUpdate.Model.Platform;
namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
@ -1851,5 +1852,28 @@ namespace skyscraper5.Skyscraper.Scraper.Storage.Filesystem
stream.Dispose();
}
public bool TestForThreemediaSegment(TarHeader tarHeader)
{
string outfilename = Path.Combine(rootDirectory.FullName, "ThreemediaOTT", tarHeader.Filename);
FileInfo fi = new FileInfo(outfilename);
return fi.Exists;
}
public void StoreThreemediaSegment(TarHeader tarHeader, MemoryStream stream)
{
string outfilename = Path.Combine(rootDirectory.FullName, "ThreemediaOTT", tarHeader.Filename);
FileInfo fi = new FileInfo(outfilename);
fi.Directory.EnsureExists();
if (stream.CanSeek)
stream.Position = 0;
FileStream fileStream = fi.OpenWrite();
stream.CopyTo(fileStream);
fileStream.Flush();
fileStream.Close();
stream.Dispose();
}
}
}

View File

@ -11,6 +11,7 @@ using skyscraper8.Experimentals.NdsSsu;
using skyscraper8.Ietf.FLUTE;
using skyscraper8.SimpleServiceDiscoveryProtocol;
using skyscraper8.Skyscraper.Drawing;
using skyscraper8.Skyscraper.Scraper.Storage.Tar;
namespace skyscraper8.Skyscraper.Scraper.Storage
{
@ -189,5 +190,15 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
{
throw new NotImplementedException();
}
public bool TestForThreemediaSegment(TarHeader tarHeader)
{
throw new NotImplementedException();
}
public void StoreThreemediaSegment(TarHeader tarHeader, MemoryStream memoryStream)
{
throw new NotImplementedException();
}
}
}

View File

@ -13,6 +13,7 @@ using skyscraper8.Experimentals.NdsSsu;
using skyscraper8.Ietf.FLUTE;
using skyscraper8.SimpleServiceDiscoveryProtocol;
using skyscraper8.Skyscraper.Drawing;
using skyscraper8.Skyscraper.Scraper.Storage.Tar;
namespace skyscraper8.Skyscraper.Scraper.Storage
{
@ -50,5 +51,7 @@ namespace skyscraper8.Skyscraper.Scraper.Storage
void StoreWneStory(uint sessionId, string filename, Stream value);
bool TestForAtsc3Segment(IPEndPoint destination, string outFileName);
void StoreAtsc3Segment(IPEndPoint destination, string outFileName, Stream stream);
bool TestForThreemediaSegment(TarHeader tarHeader);
void StoreThreemediaSegment(TarHeader tarHeader, MemoryStream memoryStream);
}
}

View File

@ -1,5 +1,7 @@
using System.Text;
using Newtonsoft.Json.Converters;
using skyscraper5.Skyscraper;
using skyscraper5.Skyscraper.IO;
namespace skyscraper8.Skyscraper.Scraper.Storage.Tar;
@ -17,6 +19,8 @@ public class TarHeader
OwnerGroupName = "sophia";
}
private TarHeader() {}
private string _filename;
public string Filename
{
@ -80,7 +84,29 @@ public class TarHeader
if (buffer.Length != 512)
throw new InvalidDataException("Invalid TAR header length");
throw new NotImplementedException();
MemoryStream ms = new MemoryStream(buffer, false);
TarHeader tarHeader = new TarHeader();
tarHeader.Filename = ms.ReadUTF8FixedLength(100).Trim('\0');
tarHeader.Permissions = new UnixFilePermissions(ms.ReadUTF8FixedLength(8));
tarHeader.Uid = (int)OctalToLong(ms.ReadUTF8FixedLength(8));
tarHeader.Gid = (int)OctalToLong(ms.ReadUTF8FixedLength(8));
tarHeader.Size = OctalToLong(ms.ReadUTF8FixedLength(12));
tarHeader.ModificationTime = OctalToLong(ms.ReadUTF8FixedLength(12)).AsUnixtime();
long checksum = OctalToLong(ms.ReadUTF8FixedLength(8));
char filetype = (char)ms.ReadUInt8();
if (buffer[257] == 'u' && buffer[258] == 's' && buffer[259] == 't' && buffer[260] == 'a' && buffer[261] == 'r')
{
ms.Position = 265;
tarHeader.OwnerUsername = ms.ReadAsciiNullTerminated();
ms.Position = 297;
tarHeader.OwnerGroupName = ms.ReadAsciiNullTerminated();
}
//TODO: actually verify the checksum!
return tarHeader;
}
public byte[] Serialize()
@ -146,6 +172,7 @@ public class TarHeader
private static long OctalToLong(string value)
{
value = value.Trim('\0');
long result = 0;
for (int i = 0; i < value.Length; ++i)

View File

@ -257,5 +257,15 @@ namespace skyscraper8.Skyscraper.Scraper.Storage.Tar
{
throw new NotImplementedException();
}
public bool TestForThreemediaSegment(TarHeader tarHeader)
{
throw new NotImplementedException();
}
public void StoreThreemediaSegment(TarHeader tarHeader, MemoryStream memoryStream)
{
throw new NotImplementedException();
}
}
}

View File

@ -12,6 +12,10 @@ public class UnixFilePermissions
public UnixFilePermissions(string permissionString)
{
if (permissionString.Length == 8)
{
permissionString = permissionString.Substring(0, 7);
}
if (permissionString.Length == 7)
permissionString = permissionString.Substring(4);

View File

@ -0,0 +1,14 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using skyscraper8.Skyscraper.Scraper.Storage.Tar;
namespace skyscraper8.ThreemediaOtt
{
internal interface IThreeMediaEventHandler
{
void OnThreeMediaFileDelivery(MemoryStream entryStream, TarHeader tarHeader, IReadOnlyDictionary<string, string> streamFooterData, ushort destinationPort);
}
}

View File

@ -53,6 +53,8 @@ namespace skyscraper8.ThreemediaOtt
throw new IOException("An attempt was made to move the file pointer before the beginning or past the end of the stream.");
}
_internalPosition = newPosition;
SeekHook();
return newPosition;

View File

@ -1,15 +1,17 @@
using System;
using log4net;
using skyscraper5.Skyscraper.IO;
using skyscraper8.Ietf.FLUTE;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using skyscraper5.Skyscraper.IO;
using skyscraper8.Ietf.FLUTE;
namespace skyscraper8.ThreemediaOtt
{
internal class ThreemediaContentFragment : IDisposable
{
private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
public void IngestPacket(LctFrame lctFrame)
{
int totalBlocks = lctFrame.FecHeader.SourceBlockLength.Value;
@ -63,14 +65,53 @@ namespace skyscraper8.ThreemediaOtt
int footerLength = (int)lastBlockStream.ReadUInt32BE();
int footerOffset = (int)(lastBlockStream.Length - 8 - footerLength);
if (footerOffset < 0)
{
logger.Info("Detected ThreeMedia Content Fragment with oversized buffer.");
//negative footer offset, will need to assemble two blocks.
byte[] lastBlockUpper = blocks[blocks.Length - 2];
byte[] lastBlockLower = blocks[blocks.Length - 1];
lastBlock = new byte[lastBlockUpper.Length + lastBlockLower.Length];
Array.Copy(lastBlockUpper,0,lastBlock,0,lastBlockUpper.Length);
Array.Copy(lastBlockLower, 0, lastBlock, lastBlockUpper.Length, lastBlockLower.Length);
footerOffset = (int)(lastBlock.Length - 8 - footerLength);
Dictionary<string, string> footer = ExtractFooter(lastBlock, footerLength, footerOffset);
int[] blockSizes = Array.ConvertAll(blocks, (x) => x.Length);
AdjustBlockSizeForFooter(blockSizes, 8 + footerLength);
ThreemediaContentFragmentStream result = new ThreemediaContentFragmentStream(blocks, blockSizes, footer);
return result;
}
else
{
//Footer is complete in one block.
Dictionary<string, string> footer = ExtractFooter(lastBlock, footerLength, footerOffset);
int[] blockSizes = Array.ConvertAll(blocks, (x) => x.Length);
blockSizes[blocks.Length - 1] = footerOffset;
//AdjustBlockSizeForFooter(blockSizes, footerOffset);
ThreemediaContentFragmentStream result = new ThreemediaContentFragmentStream(blocks, blockSizes, footer);
return result;
}
throw new NotImplementedException();
}
private void AdjustBlockSizeForFooter(int[] blockSizes, int footerOffset)
{
int currentItem = blockSizes.Length - 1;
while (footerOffset > 0)
{
int edible = Math.Min(footerOffset, blockSizes[currentItem]);
footerOffset -= edible;
blockSizes[currentItem] -= edible;
if (footerOffset > 0)
{
currentItem--;
}
}
}
private Dictionary<string, string> ExtractFooter(byte[] lastBlock, int footerLength, int footerOffset)

View File

@ -10,6 +10,7 @@ namespace skyscraper8.ThreemediaOtt
{
private readonly byte[][] _blocks;
private readonly int[] _blockSizes;
private IReadOnlyDictionary<string, string> footer;
private int _currentBlockIndex;
private int _currentBlockPosition;
@ -26,11 +27,53 @@ namespace skyscraper8.ThreemediaOtt
// Calculate total stream length based on the valid payloads
SetLength(_blockSizes.Sum(x => (long)x));
this.footer = footer;
}
public override int Read(byte[] buffer, int offset, int count)
{
throw new NotImplementedException();
if (buffer == null)
throw new ArgumentNullException(nameof(buffer));
if (offset < 0 || count < 0 || offset + count > buffer.Length)
throw new ArgumentException("Invalid buffer offset/count.");
if (_internalPosition >= Length || count == 0)
return 0;
int totalBytesRead = 0;
while (count > 0 && _currentBlockIndex < _blocks.Length)
{
int remainingInBlock = _blockSizes[_currentBlockIndex] - _currentBlockPosition;
// If the current block is fully read, move to the next one
if (remainingInBlock <= 0)
{
_currentBlockIndex++;
_currentBlockPosition = 0;
continue;
}
// Determine how much we can actually copy from the current block
int bytesToCopy = Math.Min(count, remainingInBlock);
Buffer.BlockCopy(
src: _blocks[_currentBlockIndex],
srcOffset: _currentBlockPosition,
dst: buffer,
dstOffset: offset + totalBytesRead,
count: bytesToCopy
);
// Update state
_currentBlockPosition += bytesToCopy;
_internalPosition += bytesToCopy;
totalBytesRead += bytesToCopy;
count -= bytesToCopy;
}
return totalBytesRead;
}
protected override void SeekHook()
@ -54,5 +97,13 @@ namespace skyscraper8.ThreemediaOtt
}
}
}
public IReadOnlyDictionary<string, string> FooterData
{
get
{
return footer;
}
}
}
}

View File

@ -4,12 +4,15 @@ using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using Ionic.Zlib;
using log4net;
using skyscraper5.Ietf.Rfc768;
using skyscraper5.Ietf.Rfc971;
using skyscraper5.Skyscraper.IO;
using skyscraper5.Skyscraper.Plugins;
using skyscraper5.Skyscraper.Scraper;
using skyscraper8.Ietf.FLUTE;
using skyscraper8.Skyscraper.Scraper.Storage.Tar;
namespace skyscraper8.ThreemediaOtt
{
@ -24,12 +27,12 @@ namespace skyscraper8.ThreemediaOtt
throw new NotImplementedException();
}
private SkyscraperContext context;
private IThreeMediaEventHandler eventHandler;
public void SetContext(DateTime? currentTime, object skyscraperContext)
{
if (context != null)
if (eventHandler == null)
{
context = skyscraperContext as SkyscraperContext;
eventHandler = skyscraperContext as IThreeMediaEventHandler;
}
}
@ -104,15 +107,55 @@ namespace skyscraper8.ThreemediaOtt
{
logger.WarnFormat(String.Format("The session on Port {0} is incomplete and can not be recovered.", destinationPort));
selectedSession.Dispose();
sessions.Remove(destinationPort);
return;
}
ThreemediaSessionStream stream = selectedSession.CreateStream();
string filetype = Path.GetExtension(stream.ContentLocation);
switch (filetype)
{
case ".tgz":
ExtractTarGz(stream,destinationPort);
stream.Dispose();
selectedSession.Dispose();
sessions.Remove(destinationPort);
return;
default:
logger.WarnFormat(String.Format("Unknown file type in ThreeMedia OTT Session: {0}", stream.ContentLocation));
break;
}
}
FileStream fs = File.OpenWrite("test.bin");
stream.CopyTo(fs);
fs.Flush(true);
fs.Close();
private void ExtractTarGz(ThreemediaSessionStream stream, ushort destinationPort)
{
GZipStream gz = new GZipStream(stream, CompressionMode.Decompress, false);
byte[] headerBuffer = new byte[512];
while (true)
{
int headerBufferResult = gz.Read(headerBuffer, 0, headerBuffer.Length);
if (headerBuffer[0] == 0)
break;
if (headerBufferResult == 0)
break;
TarHeader tarHeader = TarHeader.Deserialize(headerBuffer);
if (tarHeader == null)
break;
long blockyFileSize = tarHeader.Size;
bool needExtraBlock = blockyFileSize % 512 != 0;
blockyFileSize /= 512;
if (needExtraBlock)
{
blockyFileSize++;
}
blockyFileSize *= 512;
byte[] entryBuffer = gz.ReadBytes(blockyFileSize);
MemoryStream entryStream = new MemoryStream(entryBuffer, 0, (int)blockyFileSize, false);
eventHandler.OnThreeMediaFileDelivery(entryStream, tarHeader, stream.FooterData, destinationPort);
}
}
public bool StopProcessingAfterThis()

View File

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using skyscraper5.Skyscraper.IO;
using skyscraper8.Ietf.FLUTE;
namespace skyscraper8.ThreemediaOtt
@ -17,6 +18,12 @@ namespace skyscraper8.ThreemediaOtt
int currentFragment = (int)lctFrame.LctHeader.TransportObjectIdentifier;
currentFragment--;
int fragmentsBound = fragments.GetUpperBound(0);
if (currentFragment > fragmentsBound)
{
return;
}
if (fragments[currentFragment] == null)
fragments[currentFragment] = new ThreemediaContentFragment();

View File

@ -8,19 +8,189 @@ namespace skyscraper8.ThreemediaOtt
{
internal class ThreemediaSessionStream : ThreemediaBaseStream
{
public ThreemediaSessionStream(ThreemediaContentFragmentStream[] fragmentStreams)
private readonly ThreemediaContentFragmentStream[] _fragments;
private int _currentFragmentIndex;
private IReadOnlyDictionary<string, string> footerData;
public ThreemediaSessionStream(ThreemediaContentFragmentStream[] fragments)
{
throw new NotImplementedException();
_fragments = fragments ?? throw new ArgumentNullException(nameof(fragments));
// Ensure none of the passed streams are null
if (_fragments.Any(f => f == null))
{
throw new ArgumentException("Fragment array cannot contain null elements.", nameof(fragments));
}
// Calculate the total combined length of all fragments
SetLength(_fragments.Sum(f => f.Length));
// Ensure all fragments start at position 0
ResetFragmentPositions();
ParseFooterData();
}
private void ParseFooterData()
{
Dictionary<string, string> result = new Dictionary<string, string>();
foreach (ThreemediaContentFragmentStream fragment in _fragments)
{
IReadOnlyDictionary<string, string> dictionary = fragment.FooterData;
foreach (KeyValuePair<string,string> key in dictionary)
{
result[key.Key] = key.Value;
}
}
footerData = result;
}
private void ResetFragmentPositions()
{
foreach (var fragment in _fragments)
{
fragment.Position = 0;
}
_currentFragmentIndex = 0;
_internalPosition = 0;
}
public override int Read(byte[] buffer, int offset, int count)
{
throw new NotImplementedException();
if (buffer == null)
throw new ArgumentNullException(nameof(buffer));
if (offset < 0 || count < 0 || offset + count > buffer.Length)
throw new ArgumentException("Invalid buffer offset/count.");
if (_internalPosition >= Length || count == 0)
return 0;
int totalBytesRead = 0;
while (count > 0 && _currentFragmentIndex < _fragments.Length)
{
ThreemediaContentFragmentStream currentFragment = _fragments[_currentFragmentIndex];
// Read from the current active fragment stream
int bytesRead = currentFragment.Read(buffer, offset + totalBytesRead, count);
if (bytesRead == 0)
{
// Current fragment is exhausted, advance to the next one
_currentFragmentIndex++;
continue;
}
totalBytesRead += bytesRead;
_internalPosition += bytesRead;
count -= bytesRead;
}
return totalBytesRead;
}
protected override void SeekHook()
{
throw new NotImplementedException();
long remainingOffset = _internalPosition;
_currentFragmentIndex = 0;
for (int i = 0; i < _fragments.Length; i++)
{
ThreemediaContentFragmentStream fragment = _fragments[i];
if (remainingOffset >= fragment.Length)
{
// This fragment is entirely behind our new position.
// Set its pointer to its end so it's ready if we backtrack later.
fragment.Position = fragment.Length;
remainingOffset -= fragment.Length;
_currentFragmentIndex++;
}
else
{
// Our target position lands right inside this fragment.
fragment.Position = remainingOffset;
_currentFragmentIndex = i;
remainingOffset = 0;
// Reset any subsequent fragments to 0 just in case we skipped forward past them earlier
for (int j = i + 1; j < _fragments.Length; j++)
{
_fragments[j].Position = 0;
}
break;
}
}
// Edge case: if we seeked exactly to the end of the entire stream
if (_currentFragmentIndex >= _fragments.Length)
{
_currentFragmentIndex = _fragments.Length;
}
}
public string ContentBase
{
get
{
return footerData?["Content-Base"];
}
}
public string ContentLocation
{
get
{
return footerData?["Content-Location"];
}
}
public int ContentFilesize
{
get
{
return Convert.ToInt32(footerData?["Content-Filesize"]);
}
}
public int ContentLength
{
get
{
return Convert.ToInt32(footerData?["Content-Length"]);
}
}
public string ContentFragment
{
get
{
return footerData?["Content-Fragment"];
}
}
public int ContentOffset
{
get
{
return Convert.ToInt32(footerData?["Content-Offset"]);
}
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
// Clean up all underlying fragment streams automatically
foreach (var fragment in _fragments)
{
fragment.Dispose();
}
}
base.Dispose(disposing);
}
public IReadOnlyDictionary<string, string> FooterData => footerData;
}
}