/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "H264.h"
#include "AnnexB.h"
#include "BitReader.h"
#include "BitWriter.h"
#include "BufferReader.h"
#include "ByteStreamsUtils.h"
#include "ByteWriter.h"
#include "MediaInfo.h"
#include "mozilla/PodOperations.h"
#include "mozilla/Result.h"
#include "mozilla/ResultExtensions.h"
#include "mozilla/Try.h"
#include <limits>
#define READSE(var, min, max) \
{ \
int32_t val = br.ReadSE(); \
if (val < min || val > max) { \
return false; \
} \
aDest.var = val; \
}
#define READUE(var, max) \
{ \
uint32_t uval = br.ReadUE(); \
if (uval > max) { \
return false; \
} \
aDest.var = uval; \
}
mozilla::LazyLogModule gH264(
"H264");
#define LOG(msg, ...) MOZ_LOG(gH264, LogLevel::Debug, (msg,
##__VA_ARGS__))
namespace mozilla {
// Default scaling lists (per spec).
// ITU H264:
// Table 7-2 – Assignment of mnemonic names to scaling list indices and
// specification of fall-back rule
static const uint8_t Default_4x4_Intra[16] = {6, 13, 13, 20, 20, 20, 28, 28,
28, 28, 32, 32, 32, 37, 37, 42};
static const uint8_t Default_4x4_Inter[16] = {10, 14, 14, 20, 20, 20, 24, 24,
24, 24, 27, 27, 27, 30, 30, 34};
static const uint8_t Default_8x8_Intra[64] = {
6, 10, 10, 13, 11, 13, 16, 16, 16, 16, 18, 18, 18, 18, 18, 23,
23, 23, 23, 23, 23, 25, 25, 25, 25, 25, 25, 25, 27, 27, 27, 27,
27, 27, 27, 27, 29, 29, 29, 29, 29, 29, 29, 31, 31, 31, 31, 31,
31, 33, 33, 33, 33, 33, 36, 36, 36, 36, 38, 38, 38, 40, 40, 42};
static const uint8_t Default_8x8_Inter[64] = {
9, 13, 13, 15, 13, 15, 17, 17, 17, 17, 19, 19, 19, 19, 19, 21,
21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 24, 24, 24, 24,
24, 24, 24, 24, 25, 25, 25, 25, 25, 25, 25, 27, 27, 27, 27, 27,
27, 28, 28, 28, 28, 28, 30, 30, 30, 30, 32, 32, 32, 33, 33, 35};
namespace detail {
static void scaling_list(BitReader& aBr, uint8_t* aScalingList,
int aSizeOfScalingList,
const uint8_t* aDefaultList,
const uint8_t* aFallbackList) {
int32_t lastScale = 8;
int32_t nextScale = 8;
int32_t deltaScale;
// (pic|seq)_scaling_list_present_flag[i]
if (!aBr.ReadBit()) {
if (aFallbackList) {
memcpy(aScalingList, aFallbackList, aSizeOfScalingList);
}
return;
}
for (
int i = 0; i < aSizeOfScalingList; i++) {
if (nextScale != 0) {
deltaScale = aBr.ReadSE();
nextScale = (lastScale + deltaScale + 256) % 256;
if (!i && !nextScale) {
memcpy(aScalingList, aDefaultList, aSizeOfScalingList);
return;
}
}
aScalingList[i] = (nextScale == 0) ? lastScale : nextScale;
lastScale = aScalingList[i];
}
}
}
// namespace detail.
template <size_t N>
static void scaling_list(BitReader& aBr, uint8_t (&aScalingList)[N],
const uint8_t (&aDefaultList)[N],
const uint8_t (&aFallbackList)[N]) {
detail::scaling_list(aBr, aScalingList, N, aDefaultList, aFallbackList);
}
template <size_t N>
static void scaling_list(BitReader& aBr, uint8_t (&aScalingList)[N],
const uint8_t (&aDefaultList)[N]) {
detail::scaling_list(aBr, aScalingList, N, aDefaultList, nullptr);
}
SPSData::SPSData() {
PodZero(
this);
// Default values when they aren't defined as per ITU-T H.264 (2014/02).
chroma_format_idc = 1;
video_format = 5;
colour_primaries = 2;
transfer_characteristics = 2;
sample_ratio = 1.0;
memset(scaling_matrix4x4, 16,
sizeof(scaling_matrix4x4));
memset(scaling_matrix8x8, 16,
sizeof(scaling_matrix8x8));
}
bool SPSData::
operator==(
const SPSData& aOther)
const {
return this->valid && aOther.valid && !memcmp(
this, &aOther,
sizeof(SPSData));
}
bool SPSData::
operator!=(
const SPSData& aOther)
const {
return !(
operator==(aOther));
}
static PrimaryID GetPrimaryID(
int aPrimary) {
if (aPrimary < 1 || aPrimary > 22 || aPrimary == 3) {
return PrimaryID::INVALID;
}
if (aPrimary > 12 && aPrimary < 22) {
return PrimaryID::INVALID;
}
return static_cast<PrimaryID>(aPrimary);
}
static TransferID GetTransferID(
int aTransfer) {
if (aTransfer < 1 || aTransfer > 18 || aTransfer == 3) {
return TransferID::INVALID;
}
return static_cast<TransferID>(aTransfer);
}
static MatrixID GetMatrixID(
int aMatrix) {
if (aMatrix < 0 || aMatrix > 11 || aMatrix == 3) {
return MatrixID::INVALID;
}
return static_cast<MatrixID>(aMatrix);
}
gfx::YUVColorSpace SPSData::ColorSpace()
const {
// Bitfield, note that guesses with higher values take precedence over
// guesses with lower values.
enum Guess {
GUESS_BT601 = 1 << 0,
GUESS_BT709 = 1 << 1,
GUESS_BT2020 = 1 << 2,
};
uint32_t guess = 0;
switch (GetPrimaryID(colour_primaries)) {
case PrimaryID::BT709:
guess |= GUESS_BT709;
break;
case PrimaryID::BT470M:
case PrimaryID::BT470BG:
case PrimaryID::SMPTE170M:
case PrimaryID::SMPTE240M:
guess |= GUESS_BT601;
break;
case PrimaryID::BT2020:
guess |= GUESS_BT2020;
break;
case PrimaryID::FILM:
case PrimaryID::SMPTEST428_1:
case PrimaryID::SMPTEST431_2:
case PrimaryID::SMPTEST432_1:
case PrimaryID::EBU_3213_E:
case PrimaryID::INVALID:
case PrimaryID::UNSPECIFIED:
break;
}
switch (GetTransferID(transfer_characteristics)) {
case TransferID::BT709:
guess |= GUESS_BT709;
break;
case TransferID::GAMMA22:
case TransferID::GAMMA28:
case TransferID::SMPTE170M:
case TransferID::SMPTE240M:
guess |= GUESS_BT601;
break;
case TransferID::BT2020_10:
case TransferID::BT2020_12:
guess |= GUESS_BT2020;
break;
case TransferID::LINEAR:
case TransferID::LOG:
case TransferID::LOG_SQRT:
case TransferID::IEC61966_2_4:
case TransferID::BT1361_ECG:
case TransferID::IEC61966_2_1:
case TransferID::SMPTEST2084:
case TransferID::SMPTEST428_1:
case TransferID::ARIB_STD_B67:
case TransferID::INVALID:
case TransferID::UNSPECIFIED:
break;
}
switch (GetMatrixID(matrix_coefficients)) {
case MatrixID::BT709:
guess |= GUESS_BT709;
break;
case MatrixID::BT470BG:
case MatrixID::SMPTE170M:
case MatrixID::SMPTE240M:
guess |= GUESS_BT601;
break;
case MatrixID::BT2020_NCL:
case MatrixID::BT2020_CL:
guess |= GUESS_BT2020;
break;
case MatrixID::RGB:
case MatrixID::FCC:
case MatrixID::YCOCG:
case MatrixID::YDZDX:
case MatrixID::INVALID:
case MatrixID::UNSPECIFIED:
break;
}
// Removes lowest bit until only a single bit remains.
while (guess & (guess - 1)) {
guess &= guess - 1;
}
if (!guess) {
// A better default to BT601 which should die a slow death.
guess = GUESS_BT709;
}
switch (guess) {
case GUESS_BT601:
return gfx::YUVColorSpace::BT601;
case GUESS_BT709:
return gfx::YUVColorSpace::BT709;
case GUESS_BT2020:
return gfx::YUVColorSpace::BT2020;
default:
MOZ_CRASH(
"not possible to get here but makes compiler happy");
}
}
gfx::ColorDepth SPSData::ColorDepth()
const {
if (bit_depth_luma_minus8 != 0 && bit_depth_luma_minus8 != 2 &&
bit_depth_luma_minus8 != 4) {
// We don't know what that is, just assume 8 bits to prevent decoding
// regressions if we ever encounter those.
return gfx::ColorDepth::COLOR_8;
}
return gfx::ColorDepthForBitDepth(bit_depth_luma_minus8 + 8);
}
// SPSNAL and SPSNALIterator do not own their data.
class SPSNAL {
public:
SPSNAL(
const uint8_t* aPtr, size_t aLength) {
MOZ_ASSERT(aPtr);
if (aLength == 0 || (*aPtr & 0x1f) != H264_NAL_SPS) {
return;
}
mDecodedNAL = H264::DecodeNALUnit(aPtr, aLength);
if (mDecodedNAL) {
mLength = BitReader::GetBitLength(mDecodedNAL);
}
}
SPSNAL() =
default;
bool IsValid()
const {
return mDecodedNAL; }
bool operator==(
const SPSNAL& aOther)
const {
if (!mDecodedNAL || !aOther.mDecodedNAL) {
return false;
}
SPSData decodedSPS1;
SPSData decodedSPS2;
if (!GetSPSData(decodedSPS1) || !aOther.GetSPSData(decodedSPS2)) {
// Couldn't decode one SPS, perform a binary comparison
if (mLength != aOther.mLength) {
return false;
}
MOZ_ASSERT(mLength / 8 <= mDecodedNAL->Length());
if (memcmp(mDecodedNAL->Elements(), aOther.mDecodedNAL->Elements(),
mLength / 8) != 0) {
return false;
}
uint32_t remaining = mLength - (mLength & ~7);
BitReader b1(mDecodedNAL->Elements() + mLength / 8, remaining);
BitReader b2(aOther.mDecodedNAL->Elements() + mLength / 8, remaining);
for (uint32_t i = 0; i < remaining; i++) {
if (b1.ReadBit() != b2.ReadBit()) {
return false;
}
}
return true;
}
return decodedSPS1 == decodedSPS2;
}
bool operator!=(
const SPSNAL& aOther)
const {
return !(
operator==(aOther)); }
bool GetSPSData(SPSData& aDest)
const {
return H264::DecodeSPS(mDecodedNAL, aDest);
}
private:
RefPtr<mozilla::MediaByteBuffer> mDecodedNAL;
uint32_t mLength = 0;
};
class SPSNALIterator {
public:
explicit SPSNALIterator(
const mozilla::MediaByteBuffer* aExtraData)
: mExtraDataPtr(aExtraData->Elements()), mReader(aExtraData) {
if (!mReader.Read(5)) {
return;
}
auto res = mReader.ReadU8();
mNumSPS = res.isOk() ? res.unwrap() & 0x1f : 0;
if (mNumSPS == 0) {
return;
}
mValid =
true;
}
SPSNALIterator&
operator++() {
if (mEOS || !mValid) {
return *
this;
}
if (--mNumSPS == 0) {
mEOS =
true;
}
auto res = mReader.ReadU16();
uint16_t length = res.isOk() ? res.unwrap() : 0;
if (length == 0 || !mReader.Read(length)) {
mEOS =
true;
}
return *
this;
}
explicit operator bool()
const {
return mValid && !mEOS; }
SPSNAL
operator*()
const {
MOZ_ASSERT(
bool(*
this));
BufferReader reader(mExtraDataPtr + mReader.Offset(), mReader.Remaining());
auto res = reader.ReadU16();
uint16_t length = res.isOk() ? res.unwrap() : 0;
const uint8_t* ptr = reader.Read(length);
if (!ptr || !length) {
return SPSNAL();
}
return SPSNAL(ptr, length);
}
private:
const uint8_t* mExtraDataPtr;
BufferReader mReader;
bool mValid =
false;
bool mEOS =
false;
uint8_t mNumSPS = 0;
};
/* static */ Result<int, nsresult> H264::ExtractSVCTemporalId(
const uint8_t* aData, size_t aLength) {
nsTArray<AnnexB::NALEntry> paramSets;
AnnexB::ParseNALEntries(Span<
const uint8_t>(aData, aLength), paramSets);
BufferReader reader(aData, aLength);
// Discard what's needed to find the correct NAL.
int i = 0;
while (paramSets[i].mSize < 4) {
i++;
}
reader.Read(paramSets[i].mOffset);
uint8_t byte;
MOZ_TRY_VAR(byte, reader.ReadU8());
uint8_t nalUnitType = byte & 0x1f;
if (nalUnitType == H264_NAL_PREFIX || nalUnitType == H264_NAL_SLICE_EXT) {
bool svcExtensionFlag =
false;
MOZ_TRY_VAR(byte, reader.ReadU8());
svcExtensionFlag = byte & 0x80;
if (svcExtensionFlag) {
// Discard the first byte, and find the temporal id in the second byte
MOZ_TRY(reader.ReadU8());
MOZ_TRY_VAR(byte, reader.ReadU8());
int temporalId = (byte & 0xE0) >> 5;
return temporalId;
}
}
return 0;
}
/* static */ already_AddRefed<mozilla::MediaByteBuffer> H264::DecodeNALUnit(
const uint8_t* aNAL, size_t aLength) {
MOZ_ASSERT(aNAL);
if (aLength < 4) {
return nullptr;
}
RefPtr<mozilla::MediaByteBuffer> rbsp =
new mozilla::MediaByteBuffer;
BufferReader reader(aNAL, aLength);
auto res = reader.ReadU8();
if (res.isErr()) {
return nullptr;
}
uint8_t nal_unit_type = res.unwrap() & 0x1f;
uint32_t nalUnitHeaderBytes = 1;
if (nal_unit_type == H264_NAL_PREFIX || nal_unit_type == H264_NAL_SLICE_EXT ||
nal_unit_type == H264_NAL_SLICE_EXT_DVC) {
bool svc_extension_flag =
false;
bool avc_3d_extension_flag =
false;
if (nal_unit_type != H264_NAL_SLICE_EXT_DVC) {
res = reader.PeekU8();
if (res.isErr()) {
return nullptr;
}
svc_extension_flag = res.unwrap() & 0x80;
}
else {
res = reader.PeekU8();
if (res.isErr()) {
return nullptr;
}
avc_3d_extension_flag = res.unwrap() & 0x80;
}
if (svc_extension_flag) {
nalUnitHeaderBytes += 3;
}
else if (avc_3d_extension_flag) {
nalUnitHeaderBytes += 2;
}
else {
nalUnitHeaderBytes += 3;
}
}
if (!reader.Read(nalUnitHeaderBytes - 1)) {
return nullptr;
}
uint32_t lastbytes = 0xffff;
while (reader.Remaining()) {
auto res = reader.ReadU8();
if (res.isErr()) {
return nullptr;
}
uint8_t byte = res.unwrap();
if ((lastbytes & 0xffff) == 0 && byte == 0x03) {
// reset last two bytes, to detect the 0x000003 sequence again.
lastbytes = 0xffff;
}
else {
rbsp->AppendElement(byte);
}
lastbytes = (lastbytes << 8) | byte;
}
return rbsp.forget();
}
// The reverse of DecodeNALUnit. To allow the distinction between Annex B (that
// uses 0x000001 as marker) and AVCC, the pattern 0x00 0x00 0x0n (where n is
// between 0 and 3) can't be found in the bytestream. A 0x03 byte is inserted
// after the second 0. Eg. 0x00 0x00 0x00 becomes 0x00 0x00 0x03 0x00
/* static */ already_AddRefed<mozilla::MediaByteBuffer> H264::EncodeNALUnit(
const uint8_t* aNAL, size_t aLength) {
MOZ_ASSERT(aNAL);
RefPtr<MediaByteBuffer> rbsp =
new MediaByteBuffer();
BufferReader reader(aNAL, aLength);
auto res = reader.ReadU8();
if (res.isErr()) {
return rbsp.forget();
}
rbsp->AppendElement(res.unwrap());
res = reader.ReadU8();
if (res.isErr()) {
return rbsp.forget();
}
rbsp->AppendElement(res.unwrap());
while ((res = reader.ReadU8()).isOk()) {
uint8_t val = res.unwrap();
if (val <= 0x03 && rbsp->ElementAt(rbsp->Length() - 2) == 0 &&
rbsp->ElementAt(rbsp->Length() - 1) == 0) {
rbsp->AppendElement(0x03);
}
rbsp->AppendElement(val);
}
return rbsp.forget();
}
static int32_t ConditionDimension(
double aValue) {
// This will exclude NaNs and too-big values.
if (aValue > 1.0 && aValue <=
double(INT32_MAX) / 2) {
return int32_t(aValue);
}
return 0;
}
/* static */
bool H264::DecodeSPS(
const mozilla::MediaByteBuffer* aSPS, SPSData& aDest) {
if (!aSPS) {
return false;
}
BitReader br(aSPS, BitReader::GetBitLength(aSPS));
aDest.profile_idc = br.ReadBits(8);
aDest.constraint_set0_flag = br.ReadBit();
aDest.constraint_set1_flag = br.ReadBit();
aDest.constraint_set2_flag = br.ReadBit();
aDest.constraint_set3_flag = br.ReadBit();
aDest.constraint_set4_flag = br.ReadBit();
aDest.constraint_set5_flag = br.ReadBit();
br.ReadBits(2);
// reserved_zero_2bits
aDest.level_idc = br.ReadBits(8);
READUE(seq_parameter_set_id, MAX_SPS_COUNT - 1);
if (aDest.profile_idc == 100 || aDest.profile_idc == 110 ||
aDest.profile_idc == 122 || aDest.profile_idc == 244 ||
aDest.profile_idc == 44 || aDest.profile_idc == 83 ||
aDest.profile_idc == 86 || aDest.profile_idc == 118 ||
aDest.profile_idc == 128 || aDest.profile_idc == 138 ||
aDest.profile_idc == 139 || aDest.profile_idc == 134) {
READUE(chroma_format_idc, 3);
if (aDest.chroma_format_idc == 3) {
aDest.separate_colour_plane_flag = br.ReadBit();
}
READUE(bit_depth_luma_minus8, 6);
READUE(bit_depth_chroma_minus8, 6);
br.ReadBit();
// qpprime_y_zero_transform_bypass_flag
aDest.seq_scaling_matrix_present_flag = br.ReadBit();
if (aDest.seq_scaling_matrix_present_flag) {
scaling_list(br, aDest.scaling_matrix4x4[0], Default_4x4_Intra,
Default_4x4_Intra);
scaling_list(br, aDest.scaling_matrix4x4[1], Default_4x4_Intra,
aDest.scaling_matrix4x4[0]);
scaling_list(br, aDest.scaling_matrix4x4[2], Default_4x4_Intra,
aDest.scaling_matrix4x4[1]);
scaling_list(br, aDest.scaling_matrix4x4[3], Default_4x4_Inter,
Default_4x4_Inter);
scaling_list(br, aDest.scaling_matrix4x4[4], Default_4x4_Inter,
aDest.scaling_matrix4x4[3]);
scaling_list(br, aDest.scaling_matrix4x4[5], Default_4x4_Inter,
aDest.scaling_matrix4x4[4]);
scaling_list(br, aDest.scaling_matrix8x8[0], Default_8x8_Intra,
Default_8x8_Intra);
scaling_list(br, aDest.scaling_matrix8x8[1], Default_8x8_Inter,
Default_8x8_Inter);
if (aDest.chroma_format_idc == 3) {
scaling_list(br, aDest.scaling_matrix8x8[2], Default_8x8_Intra,
aDest.scaling_matrix8x8[0]);
scaling_list(br, aDest.scaling_matrix8x8[3], Default_8x8_Inter,
aDest.scaling_matrix8x8[1]);
scaling_list(br, aDest.scaling_matrix8x8[4], Default_8x8_Intra,
aDest.scaling_matrix8x8[2]);
scaling_list(br, aDest.scaling_matrix8x8[5], Default_8x8_Inter,
aDest.scaling_matrix8x8[3]);
}
}
}
else if (aDest.profile_idc == 183) {
aDest.chroma_format_idc = 0;
}
else {
// default value if chroma_format_idc isn't set.
aDest.chroma_format_idc = 1;
}
READUE(log2_max_frame_num, 12);
aDest.log2_max_frame_num += 4;
READUE(pic_order_cnt_type, 2);
if (aDest.pic_order_cnt_type == 0) {
READUE(log2_max_pic_order_cnt_lsb, 12);
aDest.log2_max_pic_order_cnt_lsb += 4;
}
else if (aDest.pic_order_cnt_type == 1) {
aDest.delta_pic_order_always_zero_flag = br.ReadBit();
READSE(offset_for_non_ref_pic, -231, 230);
READSE(offset_for_top_to_bottom_field, -231, 230);
uint32_t num_ref_frames_in_pic_order_cnt_cycle = br.ReadUE();
for (uint32_t i = 0; i < num_ref_frames_in_pic_order_cnt_cycle; i++) {
br.ReadSE();
// offset_for_ref_frame[i]
}
}
aDest.max_num_ref_frames = br.ReadUE();
aDest.gaps_in_frame_num_allowed_flag = br.ReadBit();
aDest.pic_width_in_mbs = br.ReadUE() + 1;
aDest.pic_height_in_map_units = br.ReadUE() + 1;
aDest.frame_mbs_only_flag = br.ReadBit();
if (!aDest.frame_mbs_only_flag) {
aDest.pic_height_in_map_units *= 2;
aDest.mb_adaptive_frame_field_flag = br.ReadBit();
}
aDest.direct_8x8_inference_flag = br.ReadBit();
aDest.frame_cropping_flag = br.ReadBit();
if (aDest.frame_cropping_flag) {
aDest.frame_crop_left_offset = br.ReadUE();
aDest.frame_crop_right_offset = br.ReadUE();
aDest.frame_crop_top_offset = br.ReadUE();
aDest.frame_crop_bottom_offset = br.ReadUE();
}
aDest.sample_ratio = 1.0f;
aDest.vui_parameters_present_flag = br.ReadBit();
if (aDest.vui_parameters_present_flag) {
if (!vui_parameters(br, aDest)) {
return false;
}
}
// Calculate common values.
uint8_t ChromaArrayType =
aDest.separate_colour_plane_flag ? 0 : aDest.chroma_format_idc;
// Calculate width.
uint32_t CropUnitX = 1;
uint32_t SubWidthC = aDest.chroma_format_idc == 3 ? 1 : 2;
if (ChromaArrayType != 0) {
CropUnitX = SubWidthC;
}
// Calculate Height
uint32_t CropUnitY = 2 - aDest.frame_mbs_only_flag;
uint32_t SubHeightC = aDest.chroma_format_idc <= 1 ? 2 : 1;
if (ChromaArrayType != 0) {
CropUnitY *= SubHeightC;
}
uint32_t width = aDest.pic_width_in_mbs * 16;
uint32_t height = aDest.pic_height_in_map_units * 16;
if (aDest.frame_crop_left_offset <=
std::numeric_limits<int32_t>::max() / 4 / CropUnitX &&
aDest.frame_crop_right_offset <=
std::numeric_limits<int32_t>::max() / 4 / CropUnitX &&
aDest.frame_crop_top_offset <=
std::numeric_limits<int32_t>::max() / 4 / CropUnitY &&
aDest.frame_crop_bottom_offset <=
std::numeric_limits<int32_t>::max() / 4 / CropUnitY &&
(aDest.frame_crop_left_offset + aDest.frame_crop_right_offset) *
CropUnitX <
width &&
(aDest.frame_crop_top_offset + aDest.frame_crop_bottom_offset) *
CropUnitY <
height) {
aDest.crop_left = aDest.frame_crop_left_offset * CropUnitX;
aDest.crop_right = aDest.frame_crop_right_offset * CropUnitX;
aDest.crop_top = aDest.frame_crop_top_offset * CropUnitY;
aDest.crop_bottom = aDest.frame_crop_bottom_offset * CropUnitY;
}
else {
// Nonsensical value, ignore them.
aDest.crop_left = aDest.crop_right = aDest.crop_top = aDest.crop_bottom = 0;
}
aDest.pic_width = width - aDest.crop_left - aDest.crop_right;
aDest.pic_height = height - aDest.crop_top - aDest.crop_bottom;
aDest.interlaced = !aDest.frame_mbs_only_flag;
// Determine display size.
if (aDest.sample_ratio > 1.0) {
// Increase the intrinsic width
aDest.display_width = ConditionDimension(
AssertedCast<
double>(aDest.pic_width) * aDest.sample_ratio);
aDest.display_height = aDest.pic_height;
}
else {
// Increase the intrinsic height
aDest.display_width = aDest.pic_width;
aDest.display_height = ConditionDimension(
AssertedCast<
double>(aDest.pic_height) / aDest.sample_ratio);
}
aDest.valid =
true;
return true;
}
/* static */
bool H264::vui_parameters(BitReader& aBr, SPSData& aDest) {
aDest.aspect_ratio_info_present_flag = aBr.ReadBit();
if (aDest.aspect_ratio_info_present_flag) {
aDest.aspect_ratio_idc = aBr.ReadBits(8);
aDest.sar_width = aDest.sar_height = 0;
// From E.2.1 VUI parameters semantics (ITU-T H.264 02/2014)
switch (aDest.aspect_ratio_idc) {
case 0:
// Unspecified
break;
case 1:
/*
1:1
7680x4320 16:9 frame without horizontal overscan
3840x2160 16:9 frame without horizontal overscan
1280x720 16:9 frame without horizontal overscan
1920x1080 16:9 frame without horizontal overscan (cropped from
1920x1088) 640x480 4:3 frame without horizontal overscan
*/
aDest.sample_ratio = 1.0f;
break;
case 2:
/*
12:11
720x576 4:3 frame with horizontal overscan
352x288 4:3 frame without horizontal overscan
*/
aDest.sample_ratio = 12.0 / 11.0;
break;
case 3:
/*
10:11
720x480 4:3 frame with horizontal overscan
352x240 4:3 frame without horizontal overscan
*/
aDest.sample_ratio = 10.0 / 11.0;
break;
case 4:
/*
16:11
720x576 16:9 frame with horizontal overscan
528x576 4:3 frame without horizontal overscan
*/
aDest.sample_ratio = 16.0 / 11.0;
break;
case 5:
/*
40:33
720x480 16:9 frame with horizontal overscan
528x480 4:3 frame without horizontal overscan
*/
aDest.sample_ratio = 40.0 / 33.0;
break;
case 6:
/*
24:11
352x576 4:3 frame without horizontal overscan
480x576 16:9 frame with horizontal overscan
*/
aDest.sample_ratio = 24.0 / 11.0;
break;
case 7:
/*
20:11
352x480 4:3 frame without horizontal overscan
480x480 16:9 frame with horizontal overscan
*/
aDest.sample_ratio = 20.0 / 11.0;
break;
case 8:
/*
32:11
352x576 16:9 frame without horizontal overscan
*/
aDest.sample_ratio = 32.0 / 11.0;
break;
case 9:
/*
80:33
352x480 16:9 frame without horizontal overscan
*/
aDest.sample_ratio = 80.0 / 33.0;
break;
case 10:
/*
18:11
480x576 4:3 frame with horizontal overscan
*/
aDest.sample_ratio = 18.0 / 11.0;
break;
case 11:
/*
15:11
480x480 4:3 frame with horizontal overscan
*/
aDest.sample_ratio = 15.0 / 11.0;
break;
case 12:
/*
64:33
528x576 16:9 frame with horizontal overscan
*/
aDest.sample_ratio = 64.0 / 33.0;
break;
case 13:
/*
160:99
528x480 16:9 frame without horizontal overscan
*/
aDest.sample_ratio = 160.0 / 99.0;
break;
case 14:
/*
4:3
1440x1080 16:9 frame without horizontal overscan
*/
aDest.sample_ratio = 4.0 / 3.0;
break;
case 15:
/*
3:2
1280x1080 16:9 frame without horizontal overscan
*/
aDest.sample_ratio = 3.2 / 2.0;
break;
case 16:
/*
2:1
960x1080 16:9 frame without horizontal overscan
*/
aDest.sample_ratio = 2.0 / 1.0;
break;
case 255:
/* Extended_SAR */
aDest.sar_width = aBr.ReadBits(16);
aDest.sar_height = aBr.ReadBits(16);
if (aDest.sar_width && aDest.sar_height) {
aDest.sample_ratio =
float(aDest.sar_width) /
float(aDest.sar_height);
}
break;
default:
break;
}
}
if (aBr.ReadBit()) {
// overscan_info_present_flag
aDest.overscan_appropriate_flag = aBr.ReadBit();
}
if (aBr.ReadBit()) {
// video_signal_type_present_flag
aDest.video_format = aBr.ReadBits(3);
aDest.video_full_range_flag = aBr.ReadBit();
aDest.colour_description_present_flag = aBr.ReadBit();
if (aDest.colour_description_present_flag) {
aDest.colour_primaries = aBr.ReadBits(8);
aDest.transfer_characteristics = aBr.ReadBits(8);
aDest.matrix_coefficients = aBr.ReadBits(8);
}
}
aDest.chroma_loc_info_present_flag = aBr.ReadBit();
if (aDest.chroma_loc_info_present_flag) {
BitReader& br = aBr;
// so that macro READUE works
READUE(chroma_sample_loc_type_top_field, 5);
READUE(chroma_sample_loc_type_bottom_field, 5);
}
bool timing_info_present_flag = aBr.ReadBit();
if (timing_info_present_flag) {
aBr.ReadBits(32);
// num_units_in_tick
aBr.ReadBits(32);
// time_scale
aBr.ReadBit();
// fixed_frame_rate_flag
}
return true;
}
/* static */
bool H264::DecodeSPSFromExtraData(
const mozilla::MediaByteBuffer* aExtraData,
SPSData& aDest) {
SPSNALIterator it(aExtraData);
if (!it) {
return false;
}
return (*it).GetSPSData(aDest);
}
/* static */
bool H264::EnsureSPSIsSane(SPSData& aSPS) {
bool valid =
true;
static const float default_aspect = 4.0f / 3.0f;
if (aSPS.sample_ratio <= 0.0f || aSPS.sample_ratio > 6.0f) {
if (aSPS.pic_width && aSPS.pic_height) {
aSPS.sample_ratio = (
float)aSPS.pic_width / (
float)aSPS.pic_height;
}
else {
aSPS.sample_ratio = default_aspect;
}
aSPS.display_width = aSPS.pic_width;
aSPS.display_height = aSPS.pic_height;
valid =
false;
}
if (aSPS.max_num_ref_frames > 16) {
aSPS.max_num_ref_frames = 16;
valid =
false;
}
return valid;
}
/* static */
uint32_t H264::ComputeMaxRefFrames(
const mozilla::MediaByteBuffer* aExtraData) {
uint32_t maxRefFrames = 4;
// Retrieve video dimensions from H264 SPS NAL.
SPSData spsdata;
if (DecodeSPSFromExtraData(aExtraData, spsdata)) {
// max_num_ref_frames determines the size of the sliding window
// we need to queue that many frames in order to guarantee proper
// pts frames ordering. Use a minimum of 4 to ensure proper playback of
// non compliant videos.
maxRefFrames =
std::min(std::max(maxRefFrames, spsdata.max_num_ref_frames + 1), 16u);
}
return maxRefFrames;
}
/* static */ H264::FrameType H264::GetFrameType(
const mozilla::MediaRawData* aSample) {
auto avcc = AVCCConfig::Parse(aSample);
if (avcc.isErr()) {
// We must have a valid AVCC frame with extradata.
return FrameType::INVALID;
}
MOZ_ASSERT(aSample->Data());
int nalLenSize = avcc.unwrap().NALUSize();
BufferReader reader(aSample->Data(), aSample->Size());
while (reader.Remaining() >= nalLenSize) {
uint32_t nalLen = 0;
switch (nalLenSize) {
case 1:
nalLen = reader.ReadU8().unwrapOr(0);
break;
case 2:
nalLen = reader.ReadU16().unwrapOr(0);
break;
case 3:
nalLen = reader.ReadU24().unwrapOr(0);
break;
case 4:
nalLen = reader.ReadU32().unwrapOr(0);
break;
default:
MOZ_ASSERT_UNREACHABLE(
"NAL length is up to 4 bytes");
}
if (!nalLen) {
continue;
}
const uint8_t* p = reader.Read(nalLen);
if (!p) {
return FrameType::INVALID;
}
int8_t nalType = AssertedCast<int8_t>(*p & 0x1f);
if (nalType == H264_NAL_IDR_SLICE) {
// IDR NAL.
return FrameType::I_FRAME;
}
if (nalType == H264_NAL_SEI) {
RefPtr<mozilla::MediaByteBuffer> decodedNAL = DecodeNALUnit(p, nalLen);
SEIRecoveryData data;
if (DecodeRecoverySEI(decodedNAL, data)) {
return FrameType::I_FRAME;
}
}
else if (nalType == H264_NAL_SLICE) {
RefPtr<mozilla::MediaByteBuffer> decodedNAL = DecodeNALUnit(p, nalLen);
if (DecodeISlice(decodedNAL)) {
return FrameType::I_FRAME;
}
}
}
return FrameType::OTHER;
}
/* static */ already_AddRefed<mozilla::MediaByteBuffer> H264::ExtractExtraData(
const mozilla::MediaRawData* aSample) {
auto avcc = AVCCConfig::Parse(aSample);
MOZ_ASSERT(avcc.isOk());
RefPtr<mozilla::MediaByteBuffer> extradata =
new mozilla::MediaByteBuffer;
// SPS content
nsTArray<uint8_t> sps;
ByteWriter<BigEndian> spsw(sps);
int numSps = 0;
// PPS content
nsTArray<uint8_t> pps;
ByteWriter<BigEndian> ppsw(pps);
int numPps = 0;
int nalLenSize = avcc.unwrap().NALUSize();
size_t sampleSize = aSample->Size();
if (aSample->mCrypto.IsEncrypted()) {
// The content is encrypted, we can only parse the non-encrypted data.
MOZ_ASSERT(aSample->mCrypto.mPlainSizes.Length() > 0);
if (aSample->mCrypto.mPlainSizes.Length() == 0 ||
aSample->mCrypto.mPlainSizes[0] > sampleSize) {
// This is invalid content.
return nullptr;
}
sampleSize = aSample->mCrypto.mPlainSizes[0];
}
BufferReader reader(aSample->Data(), sampleSize);
nsTArray<SPSData> SPSTable;
// If we encounter SPS with the same id but different content, we will stop
// attempting to detect duplicates.
bool checkDuplicate =
true;
// Find SPS and PPS NALUs in AVCC data
while (reader.Remaining() > nalLenSize) {
uint32_t nalLen = 0;
switch (nalLenSize) {
case 1:
Unused << reader.ReadU8().map(
[&](uint8_t x)
mutable {
return nalLen = x; });
break;
case 2:
Unused << reader.ReadU16().map(
[&](uint16_t x)
mutable {
return nalLen = x; });
break;
case 3:
Unused << reader.ReadU24().map(
[&](uint32_t x)
mutable {
return nalLen = x; });
break;
case 4:
Unused << reader.ReadU32().map(
[&](uint32_t x)
mutable {
return nalLen = x; });
break;
default:
MOZ_ASSERT_UNREACHABLE(
"NAL length size is at most 4 bytes");
}
const uint8_t* p = reader.Read(nalLen);
if (!p) {
// The read failed, but we may already have some SPS + PPS data so
// break out of reading and process what we have, if any.
break;
}
uint8_t nalType = *p & 0x1f;
if (nalType == H264_NAL_SPS) {
RefPtr<mozilla::MediaByteBuffer> sps = DecodeNALUnit(p, nalLen);
SPSData data;
if (!DecodeSPS(sps, data)) {
// Invalid SPS, ignore.
continue;
}
uint8_t spsId = data.seq_parameter_set_id;
if (spsId >= SPSTable.Length()) {
if (!SPSTable.SetLength(spsId + 1, fallible)) {
// OOM.
return nullptr;
}
}
if (checkDuplicate && SPSTable[spsId].valid && SPSTable[spsId] == data) {
// Duplicate ignore.
continue;
}
if (SPSTable[spsId].valid) {
// We already have detected a SPS with this Id. Just to be safe we
// disable SPS duplicate detection.
checkDuplicate =
false;
}
else {
SPSTable[spsId] = data;
}
numSps++;
if (!spsw.WriteU16(nalLen) || !spsw.Write(p, nalLen)) {
return extradata.forget();
}
}
else if (nalType == H264_NAL_PPS) {
numPps++;
if (!ppsw.WriteU16(nalLen) || !ppsw.Write(p, nalLen)) {
return extradata.forget();
}
}
}
// We ignore PPS data if we didn't find a SPS as we would be unable to
// decode it anyway.
numPps = numSps ? numPps : 0;
if (numSps && sps.Length() > 5) {
extradata->AppendElement(1);
// version
extradata->AppendElement(sps[3]);
// profile
extradata->AppendElement(sps[4]);
// profile compat
extradata->AppendElement(sps[5]);
// level
extradata->AppendElement(0xfc | 3);
// nal size - 1
extradata->AppendElement(0xe0 | numSps);
extradata->AppendElements(sps.Elements(), sps.Length());
extradata->AppendElement(numPps);
if (numPps) {
extradata->AppendElements(pps.Elements(), pps.Length());
}
}
return extradata.forget();
}
/* static */
uint8_t H264::NumSPS(
const mozilla::MediaByteBuffer* aExtraData) {
auto avcc = AVCCConfig::Parse(aExtraData);
return avcc.isErr() ? 0 : avcc.unwrap().mNumSPS;
}
/* static */
bool H264::HasSPS(
const mozilla::MediaByteBuffer* aExtraData) {
return H264::NumSPS(aExtraData) > 0;
}
/* static */
bool H264::CompareExtraData(
const mozilla::MediaByteBuffer* aExtraData1,
const mozilla::MediaByteBuffer* aExtraData2) {
if (aExtraData1 == aExtraData2) {
return true;
}
uint8_t numSPS = NumSPS(aExtraData1);
if (numSPS == 0 || numSPS != NumSPS(aExtraData2)) {
return false;
}
// We only compare if the SPS are the same as the various H264 decoders can
// deal with in-band change of PPS.
SPSNALIterator it1(aExtraData1);
SPSNALIterator it2(aExtraData2);
while (it1 && it2) {
if (*it1 != *it2) {
return false;
}
++it1;
++it2;
}
return true;
}
static inline Result<Ok, nsresult> ReadSEIInt(BufferReader& aBr,
uint32_t& aOutput) {
uint8_t tmpByte;
aOutput = 0;
MOZ_TRY_VAR(tmpByte, aBr.ReadU8());
while (tmpByte == 0xFF) {
aOutput += 255;
MOZ_TRY_VAR(tmpByte, aBr.ReadU8());
}
aOutput += tmpByte;
// this is the last byte
return Ok();
}
/* static */
bool H264::DecodeISlice(
const mozilla::MediaByteBuffer* aSlice) {
if (!aSlice) {
return false;
}
// According to ITU-T Rec H.264 Table 7.3.3, read the slice type from
// slice_header, and the slice type 2 and 7 are representing I slice.
BitReader br(aSlice);
// Skip `first_mb_in_slice`
br.ReadUE();
// The value of slice type can go from 0 to 9, but the value between 5 to
// 9 are actually equal to 0 to 4.
const uint32_t sliceType = br.ReadUE() % 5;
return sliceType == SLICE_TYPES::I_SLICE || sliceType == SI_SLICE;
}
/* static */
bool H264::DecodeRecoverySEI(
const mozilla::MediaByteBuffer* aSEI,
SEIRecoveryData& aDest) {
if (!aSEI) {
return false;
}
// sei_rbsp() as per 7.3.2.3 Supplemental enhancement information RBSP syntax
BufferReader br(aSEI);
do {
// sei_message() as per
// 7.3.2.3.1 Supplemental enhancement information message syntax
uint32_t payloadType = 0;
if (ReadSEIInt(br, payloadType).isErr()) {
return false;
}
uint32_t payloadSize = 0;
if (ReadSEIInt(br, payloadSize).isErr()) {
return false;
}
// sei_payload(payloadType, payloadSize) as per
// D.1 SEI payload syntax.
const uint8_t* p = br.Read(payloadSize);
if (!p) {
return false;
}
if (payloadType == 6) {
// SEI_RECOVERY_POINT
if (payloadSize == 0) {
// Invalid content, ignore.
continue;
}
// D.1.7 Recovery point SEI message syntax
BitReader br(p, payloadSize * 8);
aDest.recovery_frame_cnt = br.ReadUE();
aDest.exact_match_flag = br.ReadBit();
aDest.broken_link_flag = br.ReadBit();
aDest.changing_slice_group_idc = br.ReadBits(2);
return true;
}
}
while (br.PeekU8().isOk() &&
br.PeekU8().unwrap() !=
0x80);
// more_rbsp_data() msg[offset] != 0x80
// ignore the trailing bits rbsp_trailing_bits();
return false;
}
/*static */ already_AddRefed<mozilla::MediaByteBuffer> H264::CreateExtraData(
uint8_t aProfile, uint8_t aConstraints, H264_LEVEL aLevel,
const gfx::IntSize& aSize) {
// SPS of a 144p video.
const uint8_t originSPS[] = {0x4d, 0x40, 0x0c, 0xe8, 0x80, 0x80, 0x9d,
0x80, 0xb5, 0x01, 0x01, 0x01, 0x40, 0x00,
0x00, 0x00, 0x40, 0x00, 0x00, 0x0f, 0x03,
0xc5, 0x0a, 0x44, 0x80};
RefPtr<MediaByteBuffer> extraData =
new MediaByteBuffer();
extraData->AppendElements(originSPS,
sizeof(originSPS));
BitReader br(extraData, BitReader::GetBitLength(extraData));
RefPtr<MediaByteBuffer> sps =
new MediaByteBuffer();
BitWriter bw(sps);
br.ReadBits(8);
// Skip original profile_idc
bw.WriteU8(aProfile);
br.ReadBits(8);
// Skip original constraint flags && reserved_zero_2bits
aConstraints =
aConstraints & ~0x3;
// Ensure reserved_zero_2bits are set to 0
bw.WriteBits(aConstraints, 8);
br.ReadBits(8);
// Skip original level_idc
bw.WriteU8(
static_cast<uint8_t>(aLevel));
bw.WriteUE(br.ReadUE());
// seq_parameter_set_id (0 stored on 1 bit)
if (aProfile == 100 || aProfile == 110 || aProfile == 122 ||
aProfile == 244 || aProfile == 44 || aProfile == 83 || aProfile == 86 ||
aProfile == 118 || aProfile == 128 || aProfile == 138 ||
aProfile == 139 || aProfile == 134) {
bw.WriteUE(1);
// chroma_format_idc -> always set to 4:2:0 chroma format
bw.WriteUE(0);
// bit_depth_luma_minus8 -> always 8 bits here
bw.WriteUE(0);
// bit_depth_chroma_minus8 -> always 8 bits here
bw.WriteBit(
false);
// qpprime_y_zero_transform_bypass_flag
bw.WriteBit(
false);
// seq_scaling_matrix_present_flag
}
bw.WriteBits(br.ReadBits(11),
11);
// log2_max_frame_num to gaps_in_frame_num_allowed_flag
// skip over original exp-golomb encoded width/height
br.ReadUE();
// skip width
br.ReadUE();
// skip height
uint32_t width = aSize.width;
uint32_t widthNeeded = width % 16 != 0 ? (width / 16 + 1) * 16 : width;
uint32_t height = aSize.height;
uint32_t heightNeeded = height % 16 != 0 ? (height / 16 + 1) * 16 : height;
bw.WriteUE(widthNeeded / 16 - 1);
bw.WriteUE(heightNeeded / 16 - 1);
bw.WriteBit(br.ReadBit());
// write frame_mbs_only_flag
bw.WriteBit(br.ReadBit());
// write direct_8x8_inference_flag;
if (widthNeeded != width || heightNeeded != height) {
// Write cropping value
bw.WriteBit(
true);
// skip frame_cropping_flag
bw.WriteUE(0);
// frame_crop_left_offset
bw.WriteUE((widthNeeded - width) / 2);
// frame_crop_right_offset
bw.WriteUE(0);
// frame_crop_top_offset
bw.WriteUE((heightNeeded - height) / 2);
// frame_crop_bottom_offset
}
else {
bw.WriteBit(
false);
// skip frame_cropping_flag
}
br.ReadBit();
// skip frame_cropping_flag;
// Write the remainings of the original sps (vui_parameters which sets an
// aspect ration of 1.0)
while (br.BitsLeft()) {
bw.WriteBit(br.ReadBit());
}
bw.CloseWithRbspTrailing();
RefPtr<MediaByteBuffer> encodedSPS =
EncodeNALUnit(sps->Elements(), sps->Length());
extraData->Clear();
const uint8_t PPS[] = {0xeb, 0xef, 0x20};
WriteExtraData(
extraData, aProfile, aConstraints,
static_cast<uint8_t>(aLevel),
Span<
const uint8_t>(encodedSPS->Elements(), encodedSPS->Length()),
Span<
const uint8_t>(PPS,
sizeof(PPS)));
return extraData.forget();
}
void H264::WriteExtraData(MediaByteBuffer* aDestExtraData,
const uint8_t aProfile,
const uint8_t aConstraints,
const uint8_t aLevel,
const Span<
const uint8_t> aSPS,
const Span<
const uint8_t> aPPS) {
aDestExtraData->AppendElement(1);
aDestExtraData->AppendElement(aProfile);
aDestExtraData->AppendElement(aConstraints);
aDestExtraData->AppendElement(aLevel);
aDestExtraData->AppendElement(3);
// nalLENSize-1
aDestExtraData->AppendElement(1);
// numPPS
uint8_t c[2];
mozilla::BigEndian::writeUint16(&c[0], aSPS.Length() + 1);
aDestExtraData->AppendElements(c, 2);
aDestExtraData->AppendElement((0x00 << 7) | (0x3 << 5) | H264_NAL_SPS);
aDestExtraData->AppendElements(aSPS.Elements(), aSPS.Length());
aDestExtraData->AppendElement(1);
// numPPS
mozilla::BigEndian::writeUint16(&c[0], aPPS.Length() + 1);
aDestExtraData->AppendElements(c, 2);
aDestExtraData->AppendElement((0x00 << 7) | (0x3 << 5) | H264_NAL_PPS);
aDestExtraData->AppendElements(aPPS.Elements(), aPPS.Length());
}
/* static */ Result<AVCCConfig, nsresult> AVCCConfig::Parse(
const mozilla::MediaRawData* aSample) {
if (!aSample || aSample->Size() < 3) {
return mozilla::Err(NS_ERROR_FAILURE);
}
if (aSample->mTrackInfo &&
!aSample->mTrackInfo->mMimeType.EqualsLiteral(
"video/avc")) {
LOG(
"Only allow 'video/avc' (mimeType=%s)",
aSample->mTrackInfo->mMimeType.get());
return mozilla::Err(NS_ERROR_FAILURE);
}
return AVCCConfig::Parse(aSample->mExtraData);
}
/* static */ Result<AVCCConfig, nsresult> AVCCConfig::Parse(
const mozilla::MediaByteBuffer* aExtraData) {
if (!aExtraData || aExtraData->Length() < 7) {
return mozilla::Err(NS_ERROR_FAILURE);
}
const auto& byteBuffer = *aExtraData;
if (byteBuffer[0] != 1) {
return mozilla::Err(NS_ERROR_FAILURE);
}
AVCCConfig avcc{};
avcc.mConfigurationVersion = byteBuffer[0];
avcc.mAVCProfileIndication = byteBuffer[1];
avcc.mProfileCompatibility = byteBuffer[2];
avcc.mAVCLevelIndication = byteBuffer[3];
avcc.mLengthSizeMinusOne = byteBuffer[4] & 0x3;
avcc.mNumSPS = byteBuffer[5] & 0x1F;
return avcc;
}
#undef READUE
#undef READSE
}
// namespace mozilla
#undef LOG
