ft/skyscraper8
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Merge pull request 'As a small exercise, made the output of Stid135BbFrameReader bit-identical to the one of pts2bbf.cpp' (#1) from pts2bbf into master
All checks were successful
🚀 Pack skyscraper8 / make-zip (push) Successful in 1m32s
Details

Browse Source 
Reviewed-on: #1
This commit is contained in:
ft 2025-10-14 09:59:05 +00:00
commit 90ac87770d
19 changed files with 654 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
.gitea/workflows/demo.yaml
View File

@ -1,45 +1,39 @@
name: Gitea Actions Demo
run-name: ${{ gitea.actor }} is testing out Gitea Actions 🚀
name: 🚀 Pack skyscraper8
run-name: 👷 ${{ gitea.actor }} is building skyscraper8
on: [push]
jobs:
Explore-Gitea-Actions:
make-zip:
runs-on: self-hosted
steps:
- run: echo "🎉 The job was automatically triggered by a ${{ gitea.event_name }} event."
- run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by Gitea!"
- run: echo "🔎 The name of your branch is ${{ gitea.ref }} and your repository is ${{ gitea.repository }}."
- name: Check out repository code
- name: 📥 Check out repository code
uses: actions/checkout@v4
- run: echo "💡 The ${{ gitea.repository }} repository has been cloned to the runner."
- run: echo "🖥️ The workflow is now ready to test your code on the runner."
- name: List files in the repository
run: |
ls ${{ gitea.workspace }}
- name: Show current working directory
working-directory: ${{ gitea.workspace }}
run: |
pwd
ls -d $(pwd -P)/*
- name: Building the main assembly...
- name: 🛠️ Building the main assembly...
working-directory: ${{ gitea.workspace }}
run: |
dotnet build /p:EnableWindowsTargeting=true
- name: Typesetting the readme file...
- name: 📄 Typesetting the readme file...
working-directory: ${{ gitea.workspace }}/skyscraper8.Manual
run: |
pdflatex skyscraper8.Manual.tex
- name: Packing the main assembly...
- name: 📦 Packing the main assembly...
working-directory: ${{ gitea.workspace }}/skyscraper8/bin/Debug/net8.0
run: |
7zz a ${{ gitea.workspace }}/skyscraper8-${{ gitea.sha }}.zip *
- name: Packing the manual...
- name: 📚 Packing the manual...
working-directory: ${{ gitea.workspace }}/skyscraper8.Manual
run: |
7zz a ${{ gitea.workspace }}/skyscraper8-${{ gitea.sha }}.zip skyscraper8.Manual.pdf
- name: Uploading the Package...
- name: ☁️ Uploading the Package...
uses: akkuman/gitea-release-action@v1
with:
name: ${{ gitea.sha }}
md5sum: true
sha256sum: true
files: |-
${{ gitea.workspace }}/skyscraper8-${{ gitea.sha }}.zip
- run: echo "🍏 This job's status is ${{ job.status }}."
${{ gitea.workspace }}/skyscraper8-${{ gitea.sha }}.zip

6
.gitignore vendored
View File

@ -124,3 +124,9 @@ imgui.ini
/GUIs/skyscraper8.UI.ImGui.MonoGame/obj
/.vs/skyscraper8/CopilotIndices/17.14.995.13737
/.vs/skyscraper8/config
/skyscraper8.Manual/skyscraper8.Manual.aux
/skyscraper8.Manual/skyscraper8.Manual.log
/skyscraper8.Manual/skyscraper8.Manual.out
/skyscraper8.Manual/skyscraper8.Manual.pdf
/skyscraper8.Manual/skyscraper8.Manual.synctex.gz
/.vs/skyscraper8/CopilotIndices/17.14.1231.31060

110
skyscraper8.Manual/skyscraper8.Manual.tex
View File

@ -5,6 +5,9 @@
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{xcolor}
\usepackage{pifont,mdframed}
\usepackage{geometry}
\geometry{
a4paper,
@ -13,14 +16,32 @@
top=20mm,
}
\usepackage{hyperref}
\hypersetup{
colorlinks=true,
linkcolor=blue,
filecolor=magenta,
urlcolor=cyan,
pdftitle={skyscraper8 Manual},
pdfpagemode=FullScreen,
}
\author{Fey}
\title{skyscraper8}
\title{skyscraper8 \\ A hobbyist project to learn about MPEG-II}
\date{October 2025}
\NewDocumentCommand{\codeword}{v}{%
\texttt{\textcolor{blue}{#1}}%
}
\newenvironment{warning}
{\par\begin{mdframed}[linewidth=2pt,linecolor=red]%
\begin{list}{}{\leftmargin=1cm
\labelwidth=\leftmargin}\item[\Large\ding{43}]\textbf{IMPORTANT:}}
{\end{list}\end{mdframed}\par}
\begin{document}
\maketitle
@ -43,6 +64,7 @@ This is the niche gap skyscraper8 intents to cover: Extracting files transmitted
\section{How to get TS into skyscraper8}
skyscraper8 is a command-line operated tool. A graphical user interface is planned, but not yet fully developed. \\
This means that you have to use the command-line interpreter of your operating system to use skyscraper8. On Windows, both cmd.exe and PowerShell are known to work well.
On Linux and macOS, any halfway recent shell should be fine.
\subsection{Using a StreamReader compatible tuner}
\textbf{This is the preferred way} and applies to tuners like the popular TBS5927 or TBS6903x.
@ -52,6 +74,24 @@ This means that you have to use the command-line interpreter of your operating s
.\skyscraper8.exe cscan tcp://127.0.0.1:6969
\end{verbatim}
\subsubsection{Working with GS using StreamReader}
There are a few caveats when working with StreamReader.dll to capture GS.
Some older versions of StreamReader.dll are known to interfere with the STiD135's BBFrame encapsulation feature. Version 1.2.4.101 is a prominent example of doing this. It changes the encapsulation PID and also seems to frame GSE frames differently. Therefore, please make sure to use a recent version of StreamReader.dll. Version 1.2.5.208 is known to work correctly.
Also, make sure to disable StreamReader.dll's internal deencapsulator. You can do this by keeping the semicolon in StreamReader.ini like this
\begin{verbatim}
;FrameMode=1
\end{verbatim}
or set it to 0 like this:
\begin{verbatim}
FrameMode=0
\end{verbatim}
Why is that? While StreamReader.dll's Deencapsulation works correctly in theory, it does not seem to check whether an encapsulated packet actually contains valid MPEG-II packets and will happily try to deencapsulate all other packets as well, causing a lot of Packets with invalid data to appear in the outputted TS.
\subsection{Using a BDA compatible tuner}
Use TSDuck's \codeword{tsp} and use the \codeword{dvb} input plugin and the \codeword{ip} output plugin.
\\ TODO: Talk about TSDuck here.
@ -60,7 +100,45 @@ Use TSDuck's \codeword{tsp} and use the \codeword{dvb} input plugin and the \cod
See the paragraph about BDA tuners. The same applies to video4linux compatible tuners. TSDuck's \codeword{tsp} can be used on Windows and Linux in the same way.
\subsection{Using a SAT\textgreater IP capable tuner}
This feature is intended for devices like the Digital Devices Octopus NET or the Kathrein EXIP 418. If you're using a PCI/PCIe tuner card or an USB Tuner with your PC, these passages about SAT\textgreater IP do not apply to you.
This feature is intended for SAT\textgreater IP server devices like the Digital Devices Octopus NET or the Kathrein EXIP 418. If you're using a PCI/PCIe tuner card or an USB Tuner with your PC, these passages about SAT\textgreater IP do not apply to you.
To use skyscraper8 with a SAT\textgreater IP device, open up a shell and use the following command scheme:
\begin{verbatim}
.\skyscraper8.exe satip IP_ADDRESS DISEQC POLARITY FREQUENCY SYSTEM SYMBOL_RATE
\end{verbatim}
Replace these variables as follows:
\\
\begin{tabular}{llp{5cm}}
\hline
IP\_ADDRESS & \parbox{13.5cm}{The IP address (for example 172.20.20.122) of your SAT\textgreater IP server, or "auto" for autodetection of a SAT\textgreater IP server.} \\
\hline
DISEQC & \parbox{13.5cm}{The DiSEqC 1.0 Port you want to use. Say "1" here for A/A, "2" for B/A, "3" for A/B, 4 for B/B} \\
\hline
POLARITY & \parbox{13.5cm}{Either "H" or "V". The SAT\textgreater IP specification also allows "L" and "R", but this is not yet implemented in skyscraper8.} \\
\hline
FREQUENCY & \parbox{13.5cm}{The frequency you want to tune to in MHz. (for example 12226)} \\
\hline
SYSTEM & \parbox{13.5cm}{SYSTEM is either "S" or "S2". The SAT\textgreater IP specification does not consider S2X, but in my testings, an S2X capable device like the Octopus NET also accepts "S2" here for S2X transponders. } \\
\hline
SYMBOL\_RATE & \parbox{13.5cm}{The symbol rate of the transponder you want to tune to in Ks. (for example 27500)} \\
\hline
\end{tabular}\\
So a valid command would be for example:
\begin{verbatim}
.\skyscraper8.exe satip 172.20.20.122 2 V 12226 S2 27500
\end{verbatim}
to tune to the Hotbird DVB-NIP demo transponder, assuming Hotbird is at DiSEqC B/A. \\
If you don't know your SAT\textgreater IP server address, the following command would accomplish the same:
\begin{verbatim}
.\skyscraper8.exe satip auto 2 V 12226 S2 27500"
\end{verbatim}
If your SAT\textgreater IP server features a STiD135 chipset, you can also catch GS/GSE with it.
\subsection{Using prerecorded TS files}
This is probably the simplest way to get a TS into skyscraper8. Just run the skyscraper8.exe from the commandline passing the path to your TS file as an argument.
@ -81,19 +159,41 @@ If you are using Microsoft Windows, you can also Drag’n’Drop a TS file onto
\section{What skyscraper8 can not do}
\begin{itemize}
\item Watch TV or Listen to Radio. Extracting files and playing back audio/video are very different things, and skyscraper8 is not designed to do the latter. If this is something you're looking for, you probably want something like ProgDVB or DVBViewer.
\item Watch TV or Listen to Radio. Extracting files and playing back audio/video are two very different things, and skyscraper8 is not designed to do the latter. If this is something you're looking for, you probably want something like ProgDVB or DVBViewer.
\item Descramble encrypted channels. This is illegal in most, if not all, jurisdictions, and I want nothing to with that.
\item Circumvent copy protections. TS contained on Blu-Ray disc are usually scrambled using AACS. Although not too difficult, bypassing it is also a complicated legal matter, which is why I prefer not doing it.
\item Descramble encrypted DOCSIS\footnote{Data-Over-Cable Service Interface Specifications} Traffic. While skyscraper8 does understand DOCSIS traffic and is perfectly capable of sniffing packets from it using tuners like the TBS-6281 SE, it will drop any scrambled packets - meaning those that have the DOCSIS privacy extension enabled.
\item Do a professional grade analysis of a TS. There are many intricacies in broadcasting systems which are out of scope for a simple file extractor developed by a mere hobbyist. If you're looking for a professional grade analyzing tool, I'd recommend you to check out the tools included in VMA Video Analyzer, Dektec's StreamXpert, GkWare's StreamGuru, or the \codeword{analyze} plugin of TSDuck.
\item Handle MPEG-DASH streams. Currently, there are two known video formats used in DVB-NIP: HLS and MPEG-DASH. While skyscraper8 is able to extract segments of both types from a TS, I do not know of an easy way to playback MPEG-DASH formatted streams. HLS on the other hand is trivial.
\item Repair broken GS. When using a non STiD135-based tuner to tune to a GS, the framing gets broken in extremely weird ways. Also, when using StreamReader.dll to tune to a GS with a STiD135 - it is imperative to disable to StreamReader.dll's deencapsulator. It often breaks the framing as well. See \hyperref[sec:gswarning]{the notice about GS in Section 2.1}
\end{itemize}
\section{How skyscraper8 was made}
skyscraper8 is developed in the C\# programming language, using Microsoft Visual Studio 2022. Therefore it requires a Microsoft .NET runtime, which is included in Microsoft Windows and freely available for most Linux distributions and Apple's macOS.
The .NET assembly of skyscraper8 is not obfuscated or protected in any way. This is on purpose. If you want to study how skyscraper8 works under the hood, I hereby allow you to use tools like RedGate's Reflector or JetBrains' dotPeek to inspect the skyscraper8.dll file. I plan to release the clear source code at a later date.
The .NET assembly of skyscraper8 is not obfuscated or protected in any way. This is on purpose. If you want to study how skyscraper8 works under the hood, I hereby allow you to use tools like RedGate's Reflector or JetBrains' dotPeek to inspect the skyscraper8.dll file. Of course you can also acquire the plain source code from my Gitea instance.\footnote{\url{https://gitea.yo3explorer.moe/ft/skyscraper8}}
This document was written in \LaTeX{}, and while I wrote it myself, I did use GPT-5 for proofreading. The proofreading of this document is the only part of skyscraper8 for which an LLM was used. No part of the actual skyscraper8 codebase itself was written by an LLM.
This document was typeset in \LaTeX{}, using the TeX Live distribution, and while I wrote it myself, I did use GPT-5 for proofreading the initial version of this document. The proofreading of this document is the only part of skyscraper8 for which an LLM was used. No part of the actual skyscraper8 codebase itself was written by an LLM.
\subsection{Personal remarks and some useless bonus information}
\subsubsection{Music!}
Like a lot of programmers, I do enjoy listening to music while working. Some programmers even put song references in their software. Like how MKVToolnix' version names are actually song names, or how BSD developer fk even put \href{https://www.fabiankeil.de/nutzloseinfos.html}{a list of albums on his website} listing what albums he listened to while making it. Although this is absolutely useless, I'd liketo do this as well. Therefore, here follows a list of musical albums I listened to while developing skyscraper8 - no claim to completeness.
\begin{itemize}
\item Bel Canto - White-Out Conditions
\item Beautiful World - In Existance
\item Kitaro - Millennia
\item Macross 82-99 - A Million Miles Away
\item Megazone 23 Vocal Collection
\item Moodswings - Moodfood
\item Opus III - Guru Mother
\item Satsuki Shibano / Yoshio Ojima - Caresse
\item SHIFT UP - Stellar Blade Original Soundtrack
\item Small Affairs - Small Affairs
\item Yoshio Ojima - Hands-Some
\item Youssou N’Dour - The Guide (Wommat)
\item Yu-Gi-Oh! Sound Duel Quarter Century Collection
\end{itemize}
\end{document}

1
skyscraper8.sln.DotSettings.user
View File

@ -1,2 +1,3 @@
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFileInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fe1ab690537c44e02a014076312b886b7b2e200_003F5a_003Fcf76af61_003FFileInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/Profiling/Configurations/=1/@EntryIndexedValue">&lt;data&gt;&lt;HostParameters type="LocalHostParameters" /&gt;&lt;Argument type="StandaloneArgument"&gt;&lt;Arguments IsNull="False"&gt;&lt;/Arguments&gt;&lt;FileName IsNull="False"&gt;&lt;/FileName&gt;&lt;WorkingDirectory IsNull="False"&gt;&lt;/WorkingDirectory&gt;&lt;Scope&gt;&lt;ProcessFilters /&gt;&lt;/Scope&gt;&lt;/Argument&gt;&lt;Info type="TimelineInfo" /&gt;&lt;CoreOptions type="CoreOptions"&gt;&lt;CoreTempPath IsNull="False"&gt;&lt;/CoreTempPath&gt;&lt;RemoteEndPoint IsNull="False"&gt;&lt;/RemoteEndPoint&gt;&lt;AdditionalEnvironmentVariables /&gt;&lt;/CoreOptions&gt;&lt;HostOptions type="HostOptions"&gt;&lt;HostTempPath IsNull="False"&gt;&lt;/HostTempPath&gt;&lt;/HostOptions&gt;&lt;/data&gt;</s:String></wpf:ResourceDictionary>

2
skyscraper8/DvbNip/DvbNipReceiver.cs
View File

@ -108,9 +108,9 @@ namespace skyscraper8.DvbNip
if (isValidXml)
{
FDTInstanceType fdtAnnouncement = FluteUtilities.UnpackFluteFdt(fluteStream);
EventHandler.FluteFileAnnouncement(fluteCoordinate.Item1, fluteCoordinate.Item2, fdtAnnouncement);
if (fdtAnnouncement != null)
{
EventHandler.FluteFileAnnouncement(fluteCoordinate.Item1, fluteCoordinate.Item2, fdtAnnouncement);
SetFileAssociations(fluteListener, fdtAnnouncement);
}
}

148
skyscraper8/Skyscraper/BbframeDeencapsulator.cs → skyscraper8/GSE/BbframeDeencapsulator.cs
View File

@ -11,9 +11,9 @@ using System.Text;
using System.Threading.Tasks;
using log4net;
namespace skyscraper8.Skyscraper
namespace skyscraper8.GSE
{
internal class BbframeDeencapsulator
internal class BbframeDeencapsulator : IBbframeDeencapsulator
{
private bool interruptedGseHem;
private MemoryStream interruptedGseHemBuffer;
@ -22,6 +22,8 @@ namespace skyscraper8.Skyscraper
private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
private bool shownNonGseWarning;
public void PushPacket(byte[] bbframe)
{
MemoryStream ms = new MemoryStream(bbframe, false);
@ -32,7 +34,7 @@ namespace skyscraper8.Skyscraper
bool ccmAcmField = (matype1 & 0x10) != 0;
bool issyi = (matype1 & 0x08) != 0;
bool npd = (matype1 & 0x04) != 0;
int ro = (matype1 & 0x03);
int ro = matype1 & 0x03;
byte matype2 = ms.ReadUInt8();
@ -42,25 +44,108 @@ namespace skyscraper8.Skyscraper
ushort syncd = ms.ReadUInt16BE();
byte crc8 = ms.ReadUInt8();
if (userPacketLength == 0 && dataFieldLength == 0)
return;
switch (tsGsField)
{
case 2:
case 1:
if (sync != 0)
{
if (!shownNonGseWarning)
{
logger.WarnFormat("This stream is a valid GS, but also contains packets which are not GSE (type 0), but of type {0} . Please share a sample of this stream!", sync);
shownNonGseWarning = true;
}
return;
}
int bytes = dataFieldLength / 8;
if (ms.GetAvailableBytes() < bytes)
return;
HandleGse(ms);
HandleContinous(ms);
break;
case 2:
int hemBytes = dataFieldLength / 8;
if (ms.GetAvailableBytes() < hemBytes)
return;
HandleGseHem(ms);
break;
default: //0 = generic packetized, 1 = generic continouus, 2 = gse, 3 = ts
logger.Warn(String.Format("Unsupported: TS/GS field says 0x{0:X2}", tsGsField));
logger.Warn(string.Format("Unsupported: TS/GS field says 0x{0:X2}, please share a sample of this stream.", tsGsField));
break;
}
}
private bool gseNeedMore;
private GseFragmentation[] gseFragmentations;
private void HandleContinous(MemoryStream ms)
{
ms.Position = 10;
while (ms.GetAvailableBytes() > 0)
{
GsePacket packet = GsePacket.Read(ms);
if (packet.IsPadding)
{
break;
}
if (!packet.Valid)
{
break;
}
if (packet.IsCompletePacket)
{
if (ValidateEthertype(packet.ProtocolType.Value))
{
MpeEventHandler.OnIpDatagram(PID, packet.GseData);
}
else
{
logger.WarnFormat("Unknown EtherType: 0x{0:X4}", packet.ProtocolType.Value);
}
continue;
}
if (packet.StartIndicator && !packet.EndIndicator)
{
if (gseFragmentations == null)
gseFragmentations = new GseFragmentation[256];
gseFragmentations[packet.FragmentId.Value] = new GseFragmentation();
gseFragmentations[packet.FragmentId.Value].AddPacket(packet);
continue;
}
if (!packet.StartIndicator && packet.EndIndicator)
{
if (gseFragmentations == null)
continue;
if (gseFragmentations[packet.FragmentId.Value] == null)
continue;
gseFragmentations[packet.FragmentId.Value].AddPacket(packet);
continue;
}
if (!packet.StartIndicator && !packet.EndIndicator)
{
if (gseFragmentations == null)
continue;
throw new NotImplementedException();
}
throw new NotImplementedException();
}
}
private bool gseNeedMore;
private MemoryStream gseAssembler;
public int PID;
private void HandleGse(MemoryStream ms)
private void HandleGseHem(MemoryStream ms)
{
ms.Position = 10;
byte syncByte = ms.ReadUInt8();
@ -68,8 +153,8 @@ namespace skyscraper8.Skyscraper
long availableBytes = ms.GetAvailableBytes();
byte[] readBytes = ms.ReadBytes(availableBytes);
GseAssemblyState assemblyState = ValidateGse(gseAssembler);
GseAssemblyState assemblyState = ValidateGse(gseAssembler, true);
if (assemblyState == GseAssemblyState.NOT_YET_BEGUN)
{
if ((syncByte & 0xc0) == 0xc0)
@ -101,24 +186,24 @@ namespace skyscraper8.Skyscraper
}
gseAssembler = null;
HandleGse(ms);
HandleGseHem(ms);
return;
}
else if (assemblyState == GseAssemblyState.BROKEN_SUSPECT_ETHERTYPE)
{
gseAssembler = null;
HandleGse(ms);
HandleGseHem(ms);
return;
}
else if (assemblyState == GseAssemblyState.BROKEN_NEGATIVE_LENGTH)
{
gseAssembler = null;
HandleGse(ms);
HandleGseHem(ms);
return;
}
else
{
throw new NotImplementedException(String.Format("Unknown GSE assembler state: {0}, sync byte 0x{1}", assemblyState, syncByte));
throw new NotImplementedException(string.Format("Unknown GSE assembler state: {0}, sync byte 0x{1}", assemblyState, syncByte));
}
}
@ -130,10 +215,11 @@ namespace skyscraper8.Skyscraper
BROKEN_NEGATIVE_LENGTH,
NEED_MORE_DATA,
BROKEN_SUSPECT_LENGTH,
BROKEN_SUSPECT_ETHERTYPE
BROKEN_SUSPECT_ETHERTYPE,
VALID_NULL_PACKET, //somehow these only show up on TBS6903x, not DD Max SX8?
}
private GseAssemblyState ValidateGse(MemoryStream ms)
private GseAssemblyState ValidateGse(MemoryStream ms, bool tryFix = false)
{
if (ms == null)
return GseAssemblyState.NOT_YET_BEGUN;
@ -142,6 +228,8 @@ namespace skyscraper8.Skyscraper
ms.Position = 0;
while (ms.GetAvailableBytes() > 2)
{
long fixPosition = ms.Position;
//GSE-Header
byte byteA = ms.ReadUInt8();
bool startIndicator = (byteA & 0x80) != 0;
@ -162,6 +250,12 @@ namespace skyscraper8.Skyscraper
//Console.WriteLine("GSE Length: {0}", gseLength);
if (gseLength > 9000)
{
if (tryFix)
{
ms.Position = fixPosition;
ms.WriteUInt8Repeat(0, (int)ms.GetAvailableBytes());
break;
}
ms.Position = backupPosition;
return GseAssemblyState.BROKEN_SUSPECT_LENGTH;
}
@ -174,6 +268,11 @@ namespace skyscraper8.Skyscraper
if (startIndicator && !endIndicator)
{
if (2 > ms.GetAvailableBytes())
{
ms.Position = backupPosition;
return GseAssemblyState.NEED_MORE_DATA;
}
ushort totalLength = ms.ReadUInt16BE();
gseLength -= 2;
}
@ -188,6 +287,12 @@ namespace skyscraper8.Skyscraper
ushort protocolType = ms.ReadUInt16BE();
if (!ValidateEthertype(protocolType))
{
if (tryFix)
{
ms.Position = fixPosition;
ms.WriteUInt8Repeat(0, (int)ms.GetAvailableBytes());
break;
}
ms.Position = backupPosition;
return GseAssemblyState.BROKEN_SUSPECT_ETHERTYPE;
}
@ -216,6 +321,12 @@ namespace skyscraper8.Skyscraper
if (gseLength < 0)
{
if (tryFix)
{
ms.Position = fixPosition;
ms.WriteUInt8Repeat(0, (int)ms.GetAvailableBytes());
break;
}
ms.Position = backupPosition;
return GseAssemblyState.BROKEN_NEGATIVE_LENGTH;
}
@ -229,6 +340,11 @@ namespace skyscraper8.Skyscraper
ms.Position += gseLength;
if (!startIndicator && endIndicator)
{
if (4 > ms.GetAvailableBytes())
{
ms.Position = backupPosition;
return GseAssemblyState.NEED_MORE_DATA;
}
uint crc32 = ms.ReadUInt32BE();
int endCrc32 = (int)ms.Position;
DvbCrc32.ValidateCrc(ms, startCrc32, endCrc32);

26
skyscraper8/GSE/BbframeDumper.cs Normal file
View File

@ -0,0 +1,26 @@
using skyscraper5.Skyscraper.IO;
namespace skyscraper8.GSE;
public class BbframeDumper : IBbframeDeencapsulator, IDisposable
{
public BbframeDumper(FileInfo file)
{
file.Directory.EnsureExists();
ourStream = file.OpenWrite();
}
private FileStream ourStream;
public void PushPacket(byte[] bbframe)
{
ourStream.Write(bbframe, 0, bbframe.Length);
}
public void Dispose()
{
ourStream.Flush();
ourStream.Close();
ourStream.Dispose();
}
}

0
skyscraper8/Skyscraper/GsEventHandler.cs → skyscraper8/GSE/GsEventHandler.cs
View File

26
skyscraper8/GSE/GseFragmentation.cs Normal file
View File

@ -0,0 +1,26 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace skyscraper8.GSE
{
internal class GseFragmentation
{
public void AddPacket(GsePacket packet)
{
if (spanningGsePackets == null)
{
spanningGsePackets = new List<GsePacket>();
TotalLength = packet.TotalLength.Value;
}
spanningGsePackets.Add(packet);
}
public ushort TotalLength { get; private set; }
private List<GsePacket> spanningGsePackets;
}
}

167
skyscraper8/GSE/GsePacket.cs Normal file
View File

@ -0,0 +1,167 @@
using skyscraper5.Skyscraper.IO;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.NetworkInformation;
using System.Text;
using System.Threading.Tasks;
using skyscraper5.Skyscraper;
namespace skyscraper8.GSE
{
internal class GsePacket : Validatable
{
public static GsePacket Read(MemoryStream ms)
{
GsePacket packet = new GsePacket();
byte byteA = ms.ReadUInt8();
packet.StartIndicator = (byteA & 0x80) != 0;
packet.EndIndicator = (byteA & 0x40) != 0;
packet.LabelIndicator = (byteA & 0x30) >> 4;
if (!packet.StartIndicator && !packet.EndIndicator && packet.LabelIndicator == 0)
{
//End of baseband frame
return packet;
}
else
{
byte byteB = ms.ReadUInt8();
int gseLength = byteA & 0x0f;
gseLength <<= 8;
gseLength += byteB;
if (!packet.StartIndicator || !packet.EndIndicator)
{
packet.FragmentId = ms.ReadUInt8();
gseLength--;
}
if (packet.StartIndicator && !packet.EndIndicator)
{
packet.TotalLength = ms.ReadUInt16BE();
gseLength -= 2;
}
if (packet.StartIndicator)
{
packet.ProtocolType = ms.ReadUInt16BE();
gseLength -= 2;
if (packet.LabelIndicator == 0)
{
packet.Label = new SixByteGseLabel(ms.ReadBytes(6));
gseLength -= 6;
}
else if (packet.LabelIndicator == 1)
{
packet.Label = new ThreeByteGseLabel(ms.ReadBytes(3));
gseLength -= 3;
}
}
if (!packet.StartIndicator && packet.EndIndicator)
gseLength -= 4;
if (gseLength < 0)
{
packet.Valid = false;
return packet;
}
if (gseLength > ms.GetAvailableBytes())
{
packet.Valid = false;
return packet;
}
packet.GseData = ms.ReadBytes(gseLength);
if (!packet.StartIndicator && packet.EndIndicator)
packet.Crc32 = ms.ReadUInt32BE();
}
packet.Valid = true;
return packet;
}
public uint? Crc32 { get; set; }
public byte[] GseData { get; set; }
public GseLabel Label { get; set; }
public ushort? ProtocolType { get; set; }
public ushort? TotalLength { get; set; }
public byte? FragmentId { get; set; }
public int LabelIndicator { get; private set; }
public bool EndIndicator { get; private set; }
public bool StartIndicator { get; private set; }
public bool IsPadding
{
get
{
return !StartIndicator && !EndIndicator && LabelIndicator == 0;
}
}
public bool IsCompletePacket
{
get
{
return StartIndicator && EndIndicator;
}
}
}
public abstract class GseLabel : Validatable
{
public abstract int LabelLength { get; }
public abstract string ToString();
}
public class ThreeByteGseLabel : GseLabel
{
public ThreeByteGseLabel(byte[] buffer)
{
if (buffer.Length != 3)
{
Valid = false;
return;
}
LabelBytes = buffer;
Valid = true;
}
public byte[] LabelBytes { get; private set; }
public override string ToString()
{
return BitConverter.ToString(LabelBytes);
}
public override int LabelLength => 3;
}
public class SixByteGseLabel : GseLabel
{
public SixByteGseLabel(byte[] buffer)
{
if (buffer.Length != 6)
{
Valid = false;
return;
}
MacAddress = new PhysicalAddress(buffer);
Valid = true;
}
public PhysicalAddress MacAddress { get; private set; }
public override string ToString()
{
return MacAddress.ToString();
}
public override int LabelLength => 6;
}
}

6
skyscraper8/GSE/IBbframeDeencapsulator.cs Normal file
View File

@ -0,0 +1,6 @@
namespace skyscraper8.GSE;
public interface IBbframeDeencapsulator
{
public void PushPacket(byte[] bbframe);
}

16
skyscraper8/GSE/NullGsEventHandler.cs Normal file
View File

@ -0,0 +1,16 @@
using skyscraper5.Skyscraper;
namespace skyscraper8.GSE;
public class NullGsEventHandler : IGsEventHandler
{
public void OnIpDatagram(int pid, byte[] payload)
{
}
public void GsIpTrafficDetected()
{
}
}

37
skyscraper8/GSE/Pts2Bbf.cs Normal file
View File

@ -0,0 +1,37 @@
using log4net;
using skyscraper5.Mpeg2;
using skyscraper5.Skyscraper.Scraper;
using skyscraper5.Skyscraper.Scraper.Storage.InMemory;
using skyscraper8.Skyscraper.Scraper.Storage;
namespace skyscraper8.GSE;
public class Pts2Bbf
{
private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
public static void Run(FileInfo file)
{
if (!file.Exists)
{
logger.Error("File not found: " + file.FullName);
return;
}
string changeExtension = Path.ChangeExtension(file.FullName, ".sbbf");
BbframeDumper dumper = new BbframeDumper(new FileInfo(changeExtension));
FileStream fileStream = file.OpenRead();
TsContext mpeg2 = new TsContext();
mpeg2.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(new NullGsEventHandler(),dumper));
DataStorage dataStorage = new InMemoryScraperStorage();
ObjectStorage objectStorage = new NullObjectStorage();
SkyscraperContext skyscraper = new SkyscraperContext(mpeg2, dataStorage, objectStorage);
skyscraper.InitalizeFilterChain();
skyscraper.IngestFromStream(fileStream);
fileStream.Close();
logger.Info("Pts2Bbf finished");
}
}

45
skyscraper8/Skyscraper/DigitalDevicesBbFrameReader.cs → skyscraper8/GSE/Stid135BbFrameReader.cs
View File

@ -6,18 +6,21 @@ using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using skyscraper8.Skyscraper;
namespace skyscraper5.src.Skyscraper
namespace skyscraper8.GSE
{
internal class DigitalDevicesBbFrameReader : ITsPacketProcessor
internal class Stid135BbFrameReader : ITsPacketProcessor
{
private IGsEventHandler mpeEventHandler;
private BbframeDeencapsulator deencapsulator;
private IBbframeDeencapsulator deencapsulator;
public DigitalDevicesBbFrameReader(IGsEventHandler mpeEventHandler)
public Stid135BbFrameReader(IGsEventHandler mpeEventHandler, IBbframeDeencapsulator deencapsulator = null)
{
if (deencapsulator == null)
deencapsulator = new BbframeDeencapsulator();
this.mpeEventHandler = mpeEventHandler;
this.deencapsulator = deencapsulator;
}
private long packetsRecovered;
@ -27,25 +30,37 @@ namespace skyscraper5.src.Skyscraper
public void PushPacket(TsPacket packet)
{
byte[] packets = packet.RawPacket;
int pid = (packets[1]);
int pid = packets[1];
pid &= 0x01f;
pid <<= 8;
pid |= (packets[2]);
pid |= packets[2];
if (pid != 0x010e)
return;
if (outbuf == null)
outbuf = new MemoryStream();
if ((packets[8] & 0xff) == 0xb8)
{
if (outbuf.Length > 0)
{
byte[] completeBbframe = outbuf.ToArray();
deencapsulator.PushPacket(completeBbframe);
outbuf = new MemoryStream();
}
//Start indicaator
outbuf.Write(packets, 8, packets[7]);
}
else
{
outbuf.Write(packets, 9, packets[7] - 1);
}
/*if ((packets[8] & 0xff) == 0xb8)
{
if (outbuf != null)
{
byte[] chi = outbuf.ToArray();
if (deencapsulator == null)
{
deencapsulator = new BbframeDeencapsulator();
deencapsulator.MpeEventHandler = mpeEventHandler;
deencapsulator.PID = pid;
}
deencapsulator.PushPacket(chi);
}
outbuf = new MemoryStream();
@ -55,7 +70,7 @@ namespace skyscraper5.src.Skyscraper
{
if (outbuf != null)
outbuf.Write(packets, 9, packets[7] - 1);
}
}*/
}
}
}

50
skyscraper8/Mpeg2/TsContext.cs
View File

@ -1,15 +1,17 @@
using System;
using log4net;
using skyscraper5.Skyscraper;
using skyscraper5.src.Mpeg2.PacketFilter;
using System;
using System.Collections.Generic;
using System.Net;
using System.Runtime.CompilerServices;
using skyscraper5.Skyscraper;
using skyscraper5.src.Mpeg2.PacketFilter;
namespace skyscraper5.Mpeg2
{
public class TsContext : ITsPacketProcessor
{
private ITsPacketProcessor[] processors;
private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
private ITsPacketProcessor[] processors;
private uint[] continuities;
private ulong[] pidPackets;
@ -42,6 +44,8 @@ namespace skyscraper5.Mpeg2
if (FilterChain == null || FilterChain.Count == 0)
throw new InvalidOperationException("The filter chain has not been initialized.");
CheckBrokenEncapsulation(packet);
for (int i = 0; i < FilterChain.Count; i++)
{
if (FilterChain[i] == null)
@ -61,6 +65,44 @@ namespace skyscraper5.Mpeg2
processors[packet.PID].PushPacket(packet);
}
private bool encapsulationBrokenConfirmed;
private bool CheckBrokenEncapsulation(TsPacket packet)
{
if (encapsulationBrokenConfirmed)
return true;
switch (packet.PID)
{
case 0x0000:
case 0x0001:
break;
default:
return false;
}
int occupiedPids = CountOccupiedPids();
if (occupiedPids < 1000)
return false;
if (packet.TSC == 0)
return false;
logger.WarnFormat("It looks like you're either having bad reception or this is a GS with broken encapsulation. This means that you either have FrameMode enabled, or are not using a STiD135.");
encapsulationBrokenConfirmed = true;
return true;
}
private int CountOccupiedPids()
{
int result = 0;
for (int i = 0; i < pidPackets.Length; i++)
{
if (pidPackets[i] > 0)
result++;
}
return result;
}
private bool EnsureContinuity(TsPacket packet)
{
//Found in ISO 13818-1.pdf, page 38

8
skyscraper8/Program.cs
View File

@ -29,6 +29,7 @@ using System.Runtime.InteropServices;
using skyscraper5.Mpeg2.Descriptors;
using skyscraper5.Skyscraper.Scraper.StreamAutodetection;
using skyscraper8;
using skyscraper8.GSE;
using skyscraper8.SatIp;
using skyscraper8.SatIp.RtspResponses;
using skyscraper8.SimpleServiceDiscoveryProtocol;
@ -306,6 +307,13 @@ namespace skyscraper5
TogglePcapConfiguration(2);
return;
}
if (args[0].ToLowerInvariant().Equals("pts2bbf"))
{
FileInfo fi = new FileInfo(args[1]);
Pts2Bbf.Run(fi);
return;
}
}
/*Passing passing = new Passing();

2
skyscraper8/Properties/launchSettings.json
View File

@ -2,7 +2,7 @@
"profiles": {
"skyscraper8": {
"commandName": "Project",
"commandLineArgs": "satip auto 1 H 11141 S2 23500",
"commandLineArgs": "cscan tcp://172.20.20.203:6969",
"remoteDebugEnabled": false
},
"Container (Dockerfile)": {

28
skyscraper8/Skyscraper/Tbs6903xGsReader.cs → skyscraper8/Skyscraper/OldStreamReaderDetector.cs
View File

@ -5,6 +5,7 @@ using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using log4net;
using skyscraper5.Dvb;
using skyscraper5.Dvb.DataBroadcasting;
using skyscraper5.Mpeg2;
@ -12,13 +13,12 @@ using skyscraper5.Skyscraper.IO;
namespace skyscraper5.Skyscraper
{
internal class Tbs6903xGsReader : ITsPacketProcessor
internal class OldStreamReaderDetector : ITsPacketProcessor
{
private readonly IGsEventHandler mpeEventHandler;
private static readonly ILog logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.Name);
public Tbs6903xGsReader(IGsEventHandler mpeEventHandler)
public OldStreamReaderDetector()
{
this.mpeEventHandler = mpeEventHandler;
}
private MemoryStream outbuf;
@ -27,13 +27,16 @@ namespace skyscraper5.Skyscraper
public void PushPacket(TsPacket packet)
{
byte[] packets = packet.RawPacket;
int pid = (packets[1]);
pid &= 0x01f;
pid <<= 8;
pid |= (packets[2]);
if (pid != 0x0118)
return;
if (annoncementDone)
return;
byte[] packets = packet.RawPacket;
int pid = (packets[1]);
pid &= 0x01f;
pid <<= 8;
pid |= (packets[2]);
if (pid != 0x0118)
return;
if ((packets[8] & 0xff) == 0xd0)
{
@ -45,10 +48,9 @@ namespace skyscraper5.Skyscraper
{
if (!annoncementDone)
{
mpeEventHandler.GsIpTrafficDetected();
logger.WarnFormat("It seems like you're using a StreamReader.dll version which interferes with the STiD135's BBFrame encapsulation. Try updating to 1.2.5.208, this is known to work.");
annoncementDone = true;
}
mpeEventHandler.OnIpDatagram(0x0118, ipPacket);
}
}
outbuf = new MemoryStream();

27
skyscraper8/Skyscraper/Scraper/SkyscraperContext.cs
View File

@ -78,6 +78,7 @@ using System.Security.Policy;
using System.Text;
using skyscraper8.Experimentals.NdsSsu;
using skyscraper8.Experimentals.OtvSsu;
using skyscraper8.GSE;
using skyscraper8.Skyscraper.Net;
using skyscraper8.Skyscraper.Scraper;
using Tsubasa.IO;
@ -293,18 +294,19 @@ namespace skyscraper5.Skyscraper.Scraper
LogEvent(SkyscraperContextEvent.StartPacketProcessing);
if (buffer[0] == 0x47 && buffer[1] == 0x41 && buffer[2] == 0x18 && (buffer[3] & 0x10) != 0 && buffer[4] == 0x00 && buffer[5] == 0x80 && buffer[6] == 0x00)
{
DvbContext.RegisterPacketProcessor(0x0118, new Tbs6903xGsReader(this));
UiJunction?.SetGseMode();
LogEvent(SkyscraperContextEvent.SpecialTsMode, "TBS 6903-X GSE Pseudo TS detected.");
DvbContext.RegisterPacketProcessor(0x0118, new OldStreamReaderDetector());
SpecialTsType = 2;
}
if (buffer[0] == 0x47 && buffer[1] == 0x41 && buffer[2] == 0x0e && (buffer[3] & 0x10) != 0 && buffer[4] == 0x00 && buffer[5] == 0x80 && buffer[6] == 0x00)
{
DvbContext.RegisterPacketProcessor(0x010e, new DigitalDevicesBbFrameReader(this));
UiJunction?.SetGseMode();
LogEvent(SkyscraperContextEvent.SpecialTsMode, "Digital Devices BBFrame Pseudo TS detected.");
SpecialTsType = 3;
}
if (!DvbContext.IsPidProcessorPresent(0x010e))
{
DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(this));
UiJunction?.SetGseMode();
LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected.");
SpecialTsType = 3;
}
}
firstPacketDone = true;
}
@ -339,10 +341,7 @@ namespace skyscraper5.Skyscraper.Scraper
{
if (!DvbContext.IsPidProcessorPresent(0x0118))
{
DvbContext.RegisterPacketProcessor(0x0118, new Tbs6903xGsReader(this));
UiJunction?.SetGseMode();
LogEvent(SkyscraperContextEvent.SpecialTsMode, "TBS 6903-X GSE Pseudo TS detected.");
SpecialTsType = 2;
DvbContext.RegisterPacketProcessor(0x0118, new OldStreamReaderDetector());
return;
}
}
@ -357,9 +356,9 @@ namespace skyscraper5.Skyscraper.Scraper
{
if (!DvbContext.IsPidProcessorPresent(0x010e))
{
DvbContext.RegisterPacketProcessor(0x010e, new DigitalDevicesBbFrameReader(this));
DvbContext.RegisterPacketProcessor(0x010e, new Stid135BbFrameReader(this));
UiJunction?.SetGseMode();
LogEvent(SkyscraperContextEvent.SpecialTsMode, "Digital Devices BBFRAME data TS detected.");
LogEvent(SkyscraperContextEvent.SpecialTsMode, "STiD135 encapsulated GS detected.");
SpecialTsType = 3;
return;
}