/*
* Copyright (c) 2019, Alliance for Open Media. All rights reserved.
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
// This is an example demonstrating how to implement a multi-layer AOM
// encoding scheme for RTC video applications.
#include <assert.h>
#include <inttypes.h>
#include <limits.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <memory>
#include "config/aom_config.h"
#if CONFIG_AV1_DECODER
#include "aom/aom_decoder.h"
#endif
#include "aom/aom_encoder.h"
#include "aom/aom_image.h"
#include "aom/aom_integer.h"
#include "aom/aomcx.h"
#include "aom_dsp/bitwriter_buffer.h"
#include "aom_ports/aom_timer.h"
#include "av1/ratectrl_rtc.h"
#include "common/args.h"
#include "common/tools_common.h"
#include "common/video_writer.h"
#include "examples/encoder_util.h"
#include "examples/multilayer_metadata.h"
#define OPTION_BUFFER_SIZE 1024
#define MAX_NUM_SPATIAL_LAYERS 4
typedef struct {
const char *output_filename;
char options[OPTION_BUFFER_SIZE];
struct AvxInputContext input_ctx[MAX_NUM_SPATIAL_LAYERS];
int speed;
int aq_mode;
int layering_mode;
int output_obu;
int decode;
int tune_content;
int show_psnr;
bool use_external_rc;
bool scale_factors_explicitly_set;
const char *multilayer_metadata_file;
} AppInput;
typedef enum {
QUANTIZER = 0,
BITRATE,
SCALE_FACTOR,
AUTO_ALT_REF,
ALL_OPTION_TYPES
} LAYER_OPTION_TYPE;
static const arg_def_t outputfile =
ARG_DEF(
"o",
"output", 1,
"Output filename");
static const arg_def_t frames_arg =
ARG_DEF(
"f",
"frames", 1,
"Number of frames to encode");
static const arg_def_t threads_arg =
ARG_DEF(
"th",
"threads", 1,
"Number of threads to use");
static const arg_def_t width_arg = ARG_DEF(
"w",
"width", 1,
"Source width");
static const arg_def_t height_arg = ARG_DEF(
"h",
"height", 1,
"Source height");
static const arg_def_t timebase_arg =
ARG_DEF(
"t",
"timebase", 1,
"Timebase (num/den)");
static const arg_def_t bitrate_arg = ARG_DEF(
"b",
"target-bitrate", 1,
"Encoding bitrate, in kilobits per second");
static const arg_def_t spatial_layers_arg =
ARG_DEF(
"sl",
"spatial-layers", 1,
"Number of spatial SVC layers");
static const arg_def_t temporal_layers_arg =
ARG_DEF(
"tl",
"temporal-layers", 1,
"Number of temporal SVC layers");
static const arg_def_t layering_mode_arg =
ARG_DEF(
"lm",
"layering-mode", 1,
"Temporal layering scheme.");
static const arg_def_t kf_dist_arg =
ARG_DEF(
"k",
"kf-dist", 1,
"Number of frames between keyframes");
static const arg_def_t scale_factors_arg =
ARG_DEF(
"r",
"scale-factors", 1,
"Scale factors (lowest to highest layer)");
static const arg_def_t min_q_arg =
ARG_DEF(NULL,
"min-q", 1,
"Minimum quantizer");
static const arg_def_t max_q_arg =
ARG_DEF(NULL,
"max-q", 1,
"Maximum quantizer");
static const arg_def_t speed_arg =
ARG_DEF(
"sp",
"speed", 1,
"Speed configuration");
static const arg_def_t aqmode_arg =
ARG_DEF(
"aq",
"aqmode", 1,
"AQ mode off/on");
static const arg_def_t bitrates_arg =
ARG_DEF(
"bl",
"bitrates", 1,
"Bitrates[spatial_layer * num_temporal_layer + temporal_layer]");
static const arg_def_t dropframe_thresh_arg =
ARG_DEF(NULL,
"drop-frame", 1,
"Temporal resampling threshold (buf %)");
static const arg_def_t error_resilient_arg =
ARG_DEF(NULL,
"error-resilient", 1,
"Error resilient flag");
static const arg_def_t output_obu_arg =
ARG_DEF(NULL,
"output-obu", 1,
"Write OBUs when set to 1. Otherwise write IVF files.");
static const arg_def_t test_decode_arg =
ARG_DEF(NULL,
"test-decode", 1,
"Attempt to test decoding the output when set to 1. Default is 1.");
static const arg_def_t psnr_arg =
ARG_DEF(NULL,
"psnr", -1,
"Show PSNR in status line.");
static const arg_def_t ext_rc_arg =
ARG_DEF(NULL,
"use-ext-rc", 0,
"Use external rate control.");
static const struct arg_enum_list tune_content_enum[] = {
{
"default", AOM_CONTENT_DEFAULT },
{
"screen", AOM_CONTENT_SCREEN },
{
"film", AOM_CONTENT_FILM },
{ NULL, 0 }
};
static const arg_def_t tune_content_arg = ARG_DEF_ENUM(
NULL,
"tune-content", 1,
"Tune content type", tune_content_enum);
#if CONFIG_CWG_E050
static const arg_def_t multilayer_metadata_file_arg =
ARG_DEF(
"ml",
"multilayer_metadata_file", 1,
"Experimental: path to multilayer metadata file");
#endif
#if CONFIG_AV1_HIGHBITDEPTH
static const struct arg_enum_list bitdepth_enum[] = { {
"8", AOM_BITS_8 },
{
"10", AOM_BITS_10 },
{ NULL, 0 } };
static const arg_def_t bitdepth_arg = ARG_DEF_ENUM(
"d",
"bit-depth", 1,
"Bit depth for codec 8 or 10. ", bitdepth_enum);
#endif // CONFIG_AV1_HIGHBITDEPTH
static const arg_def_t *svc_args[] = {
&frames_arg,
&outputfile,
&width_arg,
&height_arg,
&timebase_arg,
&bitrate_arg,
&spatial_layers_arg,
&kf_dist_arg,
&scale_factors_arg,
&min_q_arg,
&max_q_arg,
&temporal_layers_arg,
&layering_mode_arg,
&threads_arg,
&aqmode_arg,
#if CONFIG_AV1_HIGHBITDEPTH
&bitdepth_arg,
#endif
&speed_arg,
&bitrates_arg,
&dropframe_thresh_arg,
&error_resilient_arg,
&output_obu_arg,
&test_decode_arg,
&tune_content_arg,
&psnr_arg,
#if CONFIG_CWG_E050
&multilayer_metadata_file_arg,
#endif
NULL,
};
#define zero(Dest) memset(&(Dest), 0,
sizeof(Dest))
static const char *exec_name;
void usage_exit(
void) {
fprintf(stderr,
"Usage: %s input_filename [input_filename ...] -o "
"output_filename\n",
exec_name);
fprintf(stderr,
"Options:\n");
arg_show_usage(stderr, svc_args);
fprintf(
stderr,
"Input files must be y4m or yuv.\n"
"If multiple input files are specified, they correspond to spatial "
"layers, and there should be as many as there are spatial layers.\n"
"All input files must have the same width, height, frame rate and number "
"of frames.\n"
"If only one file is specified, it is used for all spatial layers.\n");
exit(EXIT_FAILURE);
}
static int file_is_y4m(
const char detect[4]) {
return memcmp(detect,
"YUV4", 4) == 0;
}
static int fourcc_is_ivf(
const char detect[4]) {
if (memcmp(detect,
"DKIF", 4) == 0) {
return 1;
}
return 0;
}
static const int option_max_values[ALL_OPTION_TYPES] = { 63, INT_MAX, INT_MAX,
1 };
static const int option_min_values[ALL_OPTION_TYPES] = { 0, 0, 1, 0 };
static void open_input_file(
struct AvxInputContext *input,
aom_chroma_sample_position_t csp) {
/* Parse certain options from the input file, if possible */
input->file = strcmp(input->filename,
"-") ? fopen(input->filename,
"rb")
: set_binary_mode(stdin);
if (!input->file) fatal(
"Failed to open input file");
if (!fseeko(input->file, 0, SEEK_END)) {
/* Input file is seekable. Figure out how long it is, so we can get
* progress info.
*/
input->length = ftello(input->file);
rewind(input->file);
}
/* Default to 1:1 pixel aspect ratio. */
input->pixel_aspect_ratio.numerator = 1;
input->pixel_aspect_ratio.denominator = 1;
/* For RAW input sources, these bytes will applied on the first frame
* in read_frame().
*/
input->detect.buf_read = fread(input->detect.buf, 1, 4, input->file);
input->detect.position = 0;
if (input->detect.buf_read == 4 && file_is_y4m(input->detect.buf)) {
if (y4m_input_open(&input->y4m, input->file, input->detect.buf, 4, csp,
input->only_i420) >= 0) {
input->file_type = FILE_TYPE_Y4M;
input->width = input->y4m.pic_w;
input->height = input->y4m.pic_h;
input->pixel_aspect_ratio.numerator = input->y4m.par_n;
input->pixel_aspect_ratio.denominator = input->y4m.par_d;
input->framerate.numerator = input->y4m.fps_n;
input->framerate.denominator = input->y4m.fps_d;
input->fmt = input->y4m.aom_fmt;
input->bit_depth =
static_cast<aom_bit_depth_t>(input->y4m.bit_depth);
}
else {
fatal(
"Unsupported Y4M stream.");
}
}
else if (input->detect.buf_read == 4 && fourcc_is_ivf(input->detect.buf)) {
fatal(
"IVF is not supported as input.");
}
else {
input->file_type = FILE_TYPE_RAW;
}
}
static aom_codec_err_t extract_option(LAYER_OPTION_TYPE type,
char *input,
int *value0,
int *value1) {
if (type == SCALE_FACTOR) {
*value0 = (
int)strtol(input, &input, 10);
if (*input++ !=
'/')
return AOM_CODEC_INVALID_PARAM;
*value1 = (
int)strtol(input, &input, 10);
if (*value0 < option_min_values[SCALE_FACTOR] ||
*value1 < option_min_values[SCALE_FACTOR] ||
*value0 > option_max_values[SCALE_FACTOR] ||
*value1 > option_max_values[SCALE_FACTOR] ||
*value0 > *value1)
// num shouldn't be greater than den
return AOM_CODEC_INVALID_PARAM;
}
else {
*value0 = atoi(input);
if (*value0 < option_min_values[type] || *value0 > option_max_values[type])
return AOM_CODEC_INVALID_PARAM;
}
return AOM_CODEC_OK;
}
static aom_codec_err_t parse_layer_options_from_string(
aom_svc_params_t *svc_params, LAYER_OPTION_TYPE type,
const char *input,
int *option0,
int *option1) {
aom_codec_err_t res = AOM_CODEC_OK;
char *input_string;
char *token;
const char *delim =
",";
int num_layers = svc_params->number_spatial_layers;
int i = 0;
if (type == BITRATE)
num_layers =
svc_params->number_spatial_layers * svc_params->number_temporal_layers;
if (input == NULL || option0 == NULL ||
(option1 == NULL && type == SCALE_FACTOR))
return AOM_CODEC_INVALID_PARAM;
const size_t input_length = strlen(input);
input_string =
reinterpret_cast<
char *>(malloc(input_length + 1));
if (input_string == NULL)
return AOM_CODEC_MEM_ERROR;
memcpy(input_string, input, input_length + 1);
token = strtok(input_string, delim);
// NOLINT
for (i = 0; i < num_layers; ++i) {
if (token != NULL) {
res = extract_option(type, token, option0 + i, option1 + i);
if (res != AOM_CODEC_OK)
break;
token = strtok(NULL, delim);
// NOLINT
}
else {
res = AOM_CODEC_INVALID_PARAM;
break;
}
}
free(input_string);
return res;
}
static void parse_command_line(
int argc,
const char **argv_,
AppInput *app_input,
aom_svc_params_t *svc_params,
aom_codec_enc_cfg_t *enc_cfg) {
struct arg arg;
char **argv = NULL;
char **argi = NULL;
char **argj = NULL;
char string_options[1024] = { 0 };
// Default settings
svc_params->number_spatial_layers = 1;
svc_params->number_temporal_layers = 1;
app_input->layering_mode = 0;
app_input->output_obu = 0;
app_input->decode = 1;
enc_cfg->g_threads = 1;
enc_cfg->rc_end_usage = AOM_CBR;
// process command line options
argv = argv_dup(argc - 1, argv_ + 1);
if (!argv) {
fprintf(stderr,
"Error allocating argument list\n");
exit(EXIT_FAILURE);
}
for (argi = argj = argv; (*argj = *argi); argi += arg.argv_step) {
arg.argv_step = 1;
if (arg_match(&arg, &outputfile, argi)) {
app_input->output_filename = arg.val;
}
else if (arg_match(&arg, &width_arg, argi)) {
enc_cfg->g_w = arg_parse_uint(&arg);
}
else if (arg_match(&arg, &height_arg, argi)) {
enc_cfg->g_h = arg_parse_uint(&arg);
}
else if (arg_match(&arg, &timebase_arg, argi)) {
enc_cfg->g_timebase = arg_parse_rational(&arg);
}
else if (arg_match(&arg, &bitrate_arg, argi)) {
enc_cfg->rc_target_bitrate = arg_parse_uint(&arg);
}
else if (arg_match(&arg, &spatial_layers_arg, argi)) {
svc_params->number_spatial_layers = arg_parse_uint(&arg);
}
else if (arg_match(&arg, &temporal_layers_arg, argi)) {
svc_params->number_temporal_layers = arg_parse_uint(&arg);
}
else if (arg_match(&arg, &speed_arg, argi)) {
app_input->speed = arg_parse_uint(&arg);
if (app_input->speed > 11) {
aom_tools_warn(
"Mapping speed %d to speed 11.\n", app_input->speed);
}
}
else if (arg_match(&arg, &aqmode_arg, argi)) {
app_input->aq_mode = arg_parse_uint(&arg);
}
else if (arg_match(&arg, &threads_arg, argi)) {
enc_cfg->g_threads = arg_parse_uint(&arg);
}
else if (arg_match(&arg, &layering_mode_arg, argi)) {
app_input->layering_mode = arg_parse_int(&arg);
}
else if (arg_match(&arg, &kf_dist_arg, argi)) {
enc_cfg->kf_min_dist = arg_parse_uint(&arg);
enc_cfg->kf_max_dist = enc_cfg->kf_min_dist;
}
else if (arg_match(&arg, &scale_factors_arg, argi)) {
aom_codec_err_t res = parse_layer_options_from_string(
svc_params, SCALE_FACTOR, arg.val, svc_params->scaling_factor_num,
svc_params->scaling_factor_den);
app_input->scale_factors_explicitly_set =
true;
if (res != AOM_CODEC_OK) {
die(
"Failed to parse scale factors: %s\n",
aom_codec_err_to_string(res));
}
}
else if (arg_match(&arg, &min_q_arg, argi)) {
enc_cfg->rc_min_quantizer = arg_parse_uint(&arg);
}
else if (arg_match(&arg, &max_q_arg, argi)) {
enc_cfg->rc_max_quantizer = arg_parse_uint(&arg);
#if CONFIG_AV1_HIGHBITDEPTH
}
else if (arg_match(&arg, &bitdepth_arg, argi)) {
enc_cfg->g_bit_depth =
static_cast<aom_bit_depth_t>(arg_parse_enum_or_int(&arg));
switch (enc_cfg->g_bit_depth) {
case AOM_BITS_8:
enc_cfg->g_input_bit_depth = 8;
enc_cfg->g_profile = 0;
break;
case AOM_BITS_10:
enc_cfg->g_input_bit_depth = 10;
enc_cfg->g_profile = 0;
break;
default:
die(
"Error: Invalid bit depth selected (%d)\n", enc_cfg->g_bit_depth);
}
#endif // CONFIG_VP9_HIGHBITDEPTH
}
else if (arg_match(&arg, &dropframe_thresh_arg, argi)) {
enc_cfg->rc_dropframe_thresh = arg_parse_uint(&arg);
}
else if (arg_match(&arg, &error_resilient_arg, argi)) {
enc_cfg->g_error_resilient = arg_parse_uint(&arg);
if (enc_cfg->g_error_resilient != 0 && enc_cfg->g_error_resilient != 1)
die(
"Invalid value for error resilient (0, 1): %d.",
enc_cfg->g_error_resilient);
}
else if (arg_match(&arg, &output_obu_arg, argi)) {
app_input->output_obu = arg_parse_uint(&arg);
if (app_input->output_obu != 0 && app_input->output_obu != 1)
die(
"Invalid value for obu output flag (0, 1): %d.",
app_input->output_obu);
}
else if (arg_match(&arg, &test_decode_arg, argi)) {
app_input->decode = arg_parse_uint(&arg);
if (app_input->decode != 0 && app_input->decode != 1)
die(
"Invalid value for test decode flag (0, 1): %d.",
app_input->decode);
}
else if (arg_match(&arg, &tune_content_arg, argi)) {
app_input->tune_content = arg_parse_enum_or_int(&arg);
printf(
"tune content %d\n", app_input->tune_content);
}
else if (arg_match(&arg, &psnr_arg, argi)) {
app_input->show_psnr = 1;
}
else if (arg_match(&arg, &ext_rc_arg, argi)) {
app_input->use_external_rc =
true;
#if CONFIG_CWG_E050
}
else if (arg_match(&arg, &multilayer_metadata_file_arg, argi)) {
app_input->multilayer_metadata_file = arg.val;
#endif
}
else {
++argj;
}
}
// Total bitrate needs to be parsed after the number of layers.
for (argi = argj = argv; (*argj = *argi); argi += arg.argv_step) {
arg.argv_step = 1;
if (arg_match(&arg, &bitrates_arg, argi)) {
aom_codec_err_t res = parse_layer_options_from_string(
svc_params, BITRATE, arg.val, svc_params->layer_target_bitrate, NULL);
if (res != AOM_CODEC_OK) {
die(
"Failed to parse bitrates: %s\n", aom_codec_err_to_string(res));
}
}
else {
++argj;
}
}
// There will be a space in front of the string options
if (strlen(string_options) > 0)
strncpy(app_input->options, string_options, OPTION_BUFFER_SIZE);
// Check for unrecognized options
for (argi = argv; *argi; ++argi)
if (argi[0][0] ==
'-' && strlen(argi[0]) > 1)
die(
"Error: Unrecognized option %s\n", *argi);
if (argv[0] == NULL) {
usage_exit();
}
int input_count = 0;
while (argv[input_count] != NULL && input_count < MAX_NUM_SPATIAL_LAYERS) {
app_input->input_ctx[input_count].filename = argv[input_count];
++input_count;
}
if (input_count > 1 && input_count != svc_params->number_spatial_layers) {
die(
"Error: Number of input files does not match number of spatial layers");
}
if (argv[input_count] != NULL) {
die(
"Error: Too many input files specified, there should be at most %d",
MAX_NUM_SPATIAL_LAYERS);
}
free(argv);
for (
int i = 0; i < input_count; ++i) {
open_input_file(&app_input->input_ctx[i], AOM_CSP_UNKNOWN);
if (app_input->input_ctx[i].file_type == FILE_TYPE_Y4M) {
if (enc_cfg->g_w == 0 || enc_cfg->g_h == 0) {
// Override these settings with the info from Y4M file.
enc_cfg->g_w = app_input->input_ctx[i].width;
enc_cfg->g_h = app_input->input_ctx[i].height;
// g_timebase is the reciprocal of frame rate.
enc_cfg->g_timebase.num = app_input->input_ctx[i].framerate.denominator;
enc_cfg->g_timebase.den = app_input->input_ctx[i].framerate.numerator;
}
else if (enc_cfg->g_w != app_input->input_ctx[i].width ||
enc_cfg->g_h != app_input->input_ctx[i].height ||
enc_cfg->g_timebase.num !=
app_input->input_ctx[i].framerate.denominator ||
enc_cfg->g_timebase.den !=
app_input->input_ctx[i].framerate.numerator) {
die(
"Error: Input file dimensions and/or frame rate mismatch");
}
}
}
if (enc_cfg->g_w == 0 || enc_cfg->g_h == 0) {
die(
"Error: Input file dimensions not set, use -w and -h");
}
if (enc_cfg->g_w < 16 || enc_cfg->g_w % 2 || enc_cfg->g_h < 16 ||
enc_cfg->g_h % 2)
die(
"Invalid resolution: %d x %d\n", enc_cfg->g_w, enc_cfg->g_h);
printf(
"Codec %s\n"
"layers: %d\n"
"width %u, height: %u\n"
"num: %d, den: %d, bitrate: %u\n"
"gop size: %u\n",
aom_codec_iface_name(aom_codec_av1_cx()),
svc_params->number_spatial_layers, enc_cfg->g_w, enc_cfg->g_h,
enc_cfg->g_timebase.num, enc_cfg->g_timebase.den,
enc_cfg->rc_target_bitrate, enc_cfg->kf_max_dist);
}
static const int mode_to_num_temporal_layers[12] = {
1, 2, 3, 3, 2, 1, 1, 3, 3, 3, 3, 3,
};
static const int mode_to_num_spatial_layers[12] = {
1, 1, 1, 1, 1, 2, 3, 2, 3, 3, 3, 3,
};
// For rate control encoding stats.
struct RateControlMetrics {
// Number of input frames per layer.
int layer_input_frames[AOM_MAX_TS_LAYERS];
// Number of encoded non-key frames per layer.
int layer_enc_frames[AOM_MAX_TS_LAYERS];
// Framerate per layer layer (cumulative).
double layer_framerate[AOM_MAX_TS_LAYERS];
// Target average frame size per layer (per-frame-bandwidth per layer).
double layer_pfb[AOM_MAX_LAYERS];
// Actual average frame size per layer.
double layer_avg_frame_size[AOM_MAX_LAYERS];
// Average rate mismatch per layer (|target - actual| / target).
double layer_avg_rate_mismatch[AOM_MAX_LAYERS];
// Actual encoding bitrate per layer (cumulative across temporal layers).
double layer_encoding_bitrate[AOM_MAX_LAYERS];
// Average of the short-time encoder actual bitrate.
// TODO(marpan): Should we add these short-time stats for each layer?
double avg_st_encoding_bitrate;
// Variance of the short-time encoder actual bitrate.
double variance_st_encoding_bitrate;
// Window (number of frames) for computing short-timee encoding bitrate.
int window_size;
// Number of window measurements.
int window_count;
int layer_target_bitrate[AOM_MAX_LAYERS];
};
static const int REF_FRAMES = 8;
static const int INTER_REFS_PER_FRAME = 7;
// Reference frames used in this example encoder.
enum {
SVC_LAST_FRAME = 0,
SVC_LAST2_FRAME,
SVC_LAST3_FRAME,
SVC_GOLDEN_FRAME,
SVC_BWDREF_FRAME,
SVC_ALTREF2_FRAME,
SVC_ALTREF_FRAME
};
static int read_frame(
struct AvxInputContext *input_ctx, aom_image_t *img) {
FILE *f = input_ctx->file;
y4m_input *y4m = &input_ctx->y4m;
int shortread = 0;
if (input_ctx->file_type == FILE_TYPE_Y4M) {
if (y4m_input_fetch_frame(y4m, f, img) < 1)
return 0;
}
else {
shortread = read_yuv_frame(input_ctx, img);
}
return !shortread;
}
static void close_input_file(
struct AvxInputContext *input) {
fclose(input->file);
if (input->file_type == FILE_TYPE_Y4M) y4m_input_close(&input->y4m);
}
// Note: these rate control metrics assume only 1 key frame in the
// sequence (i.e., first frame only). So for temporal pattern# 7
// (which has key frame for every frame on base layer), the metrics
// computation will be off/wrong.
// TODO(marpan): Update these metrics to account for multiple key frames
// in the stream.
static void set_rate_control_metrics(
struct RateControlMetrics *rc,
double framerate,
int ss_number_layers,
int ts_number_layers) {
int ts_rate_decimator[AOM_MAX_TS_LAYERS] = { 1 };
ts_rate_decimator[0] = 1;
if (ts_number_layers == 2) {
ts_rate_decimator[0] = 2;
ts_rate_decimator[1] = 1;
}
if (ts_number_layers == 3) {
ts_rate_decimator[0] = 4;
ts_rate_decimator[1] = 2;
ts_rate_decimator[2] = 1;
}
// Set the layer (cumulative) framerate and the target layer (non-cumulative)
// per-frame-bandwidth, for the rate control encoding stats below.
for (
int sl = 0; sl < ss_number_layers; ++sl) {
int i = sl * ts_number_layers;
rc->layer_framerate[0] = framerate / ts_rate_decimator[0];
rc->layer_pfb[i] =
1000.0 * rc->layer_target_bitrate[i] / rc->layer_framerate[0];
for (
int tl = 0; tl < ts_number_layers; ++tl) {
i = sl * ts_number_layers + tl;
if (tl > 0) {
rc->layer_framerate[tl] = framerate / ts_rate_decimator[tl];
rc->layer_pfb[i] =
1000.0 *
(rc->layer_target_bitrate[i] - rc->layer_target_bitrate[i - 1]) /
(rc->layer_framerate[tl] - rc->layer_framerate[tl - 1]);
}
rc->layer_input_frames[tl] = 0;
rc->layer_enc_frames[tl] = 0;
rc->layer_encoding_bitrate[i] = 0.0;
rc->layer_avg_frame_size[i] = 0.0;
rc->layer_avg_rate_mismatch[i] = 0.0;
}
}
rc->window_count = 0;
rc->window_size = 15;
rc->avg_st_encoding_bitrate = 0.0;
rc->variance_st_encoding_bitrate = 0.0;
}
static void printout_rate_control_summary(
struct RateControlMetrics *rc,
int frame_cnt,
int ss_number_layers,
int ts_number_layers) {
int tot_num_frames = 0;
double perc_fluctuation = 0.0;
printf(
"Total number of processed frames: %d\n\n", frame_cnt - 1);
printf(
"Rate control layer stats for %d layer(s):\n\n", ts_number_layers);
for (
int sl = 0; sl < ss_number_layers; ++sl) {
tot_num_frames = 0;
for (
int tl = 0; tl < ts_number_layers; ++tl) {
int i = sl * ts_number_layers + tl;
const int num_dropped =
tl > 0 ? rc->layer_input_frames[tl] - rc->layer_enc_frames[tl]
: rc->layer_input_frames[tl] - rc->layer_enc_frames[tl] - 1;
tot_num_frames += rc->layer_input_frames[tl];
rc->layer_encoding_bitrate[i] = 0.001 * rc->layer_framerate[tl] *
rc->layer_encoding_bitrate[i] /
tot_num_frames;
rc->layer_avg_frame_size[i] =
rc->layer_avg_frame_size[i] / rc->layer_enc_frames[tl];
rc->layer_avg_rate_mismatch[i] =
100.0 * rc->layer_avg_rate_mismatch[i] / rc->layer_enc_frames[tl];
printf(
"For layer#: %d %d \n", sl, tl);
printf(
"Bitrate (target vs actual): %d %f\n", rc->layer_target_bitrate[i],
rc->layer_encoding_bitrate[i]);
printf(
"Average frame size (target vs actual): %f %f\n", rc->layer_pfb[i],
rc->layer_avg_frame_size[i]);
printf(
"Average rate_mismatch: %f\n", rc->layer_avg_rate_mismatch[i]);
printf(
"Number of input frames, encoded (non-key) frames, "
"and perc dropped frames: %d %d %f\n",
rc->layer_input_frames[tl], rc->layer_enc_frames[tl],
100.0 * num_dropped / rc->layer_input_frames[tl]);
printf(
"\n");
}
}
rc->avg_st_encoding_bitrate = rc->avg_st_encoding_bitrate / rc->window_count;
rc->variance_st_encoding_bitrate =
rc->variance_st_encoding_bitrate / rc->window_count -
(rc->avg_st_encoding_bitrate * rc->avg_st_encoding_bitrate);
perc_fluctuation = 100.0 * sqrt(rc->variance_st_encoding_bitrate) /
rc->avg_st_encoding_bitrate;
printf(
"Short-time stats, for window of %d frames:\n", rc->window_size);
printf(
"Average, rms-variance, and percent-fluct: %f %f %f\n",
rc->avg_st_encoding_bitrate, sqrt(rc->variance_st_encoding_bitrate),
perc_fluctuation);
if (frame_cnt - 1 != tot_num_frames)
die(
"Error: Number of input frames not equal to output!\n");
}
// Layer pattern configuration.
static void set_layer_pattern(
int layering_mode,
int superframe_cnt, aom_svc_layer_id_t *layer_id,
aom_svc_ref_frame_config_t *ref_frame_config,
aom_svc_ref_frame_comp_pred_t *ref_frame_comp_pred,
int *use_svc_control,
int spatial_layer_id,
int is_key_frame,
int ksvc_mode,
int speed) {
// Setting this flag to 1 enables simplex example of
// RPS (Reference Picture Selection) for 1 layer.
int use_rps_example = 0;
int i;
int enable_longterm_temporal_ref = 1;
int shift = (layering_mode == 8) ? 2 : 0;
int simulcast_mode = (layering_mode == 11);
*use_svc_control = 1;
layer_id->spatial_layer_id = spatial_layer_id;
int lag_index = 0;
int base_count = superframe_cnt >> 2;
ref_frame_comp_pred->use_comp_pred[0] = 0;
// GOLDEN_LAST
ref_frame_comp_pred->use_comp_pred[1] = 0;
// LAST2_LAST
ref_frame_comp_pred->use_comp_pred[2] = 0;
// ALTREF_LAST
// Set the reference map buffer idx for the 7 references:
// LAST_FRAME (0), LAST2_FRAME(1), LAST3_FRAME(2), GOLDEN_FRAME(3),
// BWDREF_FRAME(4), ALTREF2_FRAME(5), ALTREF_FRAME(6).
for (i = 0; i < INTER_REFS_PER_FRAME; i++) ref_frame_config->ref_idx[i] = i;
for (i = 0; i < INTER_REFS_PER_FRAME; i++) ref_frame_config->reference[i] = 0;
for (i = 0; i < REF_FRAMES; i++) ref_frame_config->refresh[i] = 0;
if (ksvc_mode) {
// Same pattern as case 9, but the reference strucutre will be constrained
// below.
layering_mode = 9;
}
switch (layering_mode) {
case 0:
if (use_rps_example == 0) {
// 1-layer: update LAST on every frame, reference LAST.
layer_id->temporal_layer_id = 0;
layer_id->spatial_layer_id = 0;
ref_frame_config->refresh[0] = 1;
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
}
else {
// Pattern of 2 references (ALTREF and GOLDEN) trailing
// LAST by 4 and 8 frames, with some switching logic to
// sometimes only predict from the longer-term reference
//(golden here). This is simple example to test RPS
// (reference picture selection).
int last_idx = 0;
int last_idx_refresh = 0;
int gld_idx = 0;
int alt_ref_idx = 0;
int lag_alt = 4;
int lag_gld = 8;
layer_id->temporal_layer_id = 0;
layer_id->spatial_layer_id = 0;
int sh = 8;
// slots 0 - 7.
// Moving index slot for last: 0 - (sh - 1)
if (superframe_cnt > 1) last_idx = (superframe_cnt - 1) % sh;
// Moving index for refresh of last: one ahead for next frame.
last_idx_refresh = superframe_cnt % sh;
// Moving index for gld_ref, lag behind current by lag_gld
if (superframe_cnt > lag_gld) gld_idx = (superframe_cnt - lag_gld) % sh;
// Moving index for alt_ref, lag behind LAST by lag_alt frames.
if (superframe_cnt > lag_alt)
alt_ref_idx = (superframe_cnt - lag_alt) % sh;
// Set the ref_idx.
// Default all references to slot for last.
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = last_idx;
// Set the ref_idx for the relevant references.
ref_frame_config->ref_idx[SVC_LAST_FRAME] = last_idx;
ref_frame_config->ref_idx[SVC_LAST2_FRAME] = last_idx_refresh;
ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = gld_idx;
ref_frame_config->ref_idx[SVC_ALTREF_FRAME] = alt_ref_idx;
// Refresh this slot, which will become LAST on next frame.
ref_frame_config->refresh[last_idx_refresh] = 1;
// Reference LAST, ALTREF, and GOLDEN
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
ref_frame_config->reference[SVC_ALTREF_FRAME] = 1;
ref_frame_config->reference[SVC_GOLDEN_FRAME] = 1;
// Switch to only GOLDEN every 300 frames.
if (superframe_cnt % 200 == 0 && superframe_cnt > 0) {
ref_frame_config->reference[SVC_LAST_FRAME] = 0;
ref_frame_config->reference[SVC_ALTREF_FRAME] = 0;
ref_frame_config->reference[SVC_GOLDEN_FRAME] = 1;
// Test if the long-term is LAST instead, this is just a renaming
// but its tests if encoder behaves the same, whether its
// LAST or GOLDEN.
if (superframe_cnt % 400 == 0 && superframe_cnt > 0) {
ref_frame_config->ref_idx[SVC_LAST_FRAME] = gld_idx;
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
ref_frame_config->reference[SVC_ALTREF_FRAME] = 0;
ref_frame_config->reference[SVC_GOLDEN_FRAME] = 0;
}
}
}
break;
case 1:
// 2-temporal layer.
// 1 3 5
// 0 2 4
// Keep golden fixed at slot 3.
base_count = superframe_cnt >> 1;
ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 3;
// Cyclically refresh slots 5, 6, 7, for lag alt ref.
lag_index = 5;
if (base_count > 0) {
lag_index = 5 + (base_count % 3);
if (superframe_cnt % 2 != 0) lag_index = 5 + ((base_count + 1) % 3);
}
// Set the altref slot to lag_index.
ref_frame_config->ref_idx[SVC_ALTREF_FRAME] = lag_index;
if (superframe_cnt % 2 == 0) {
layer_id->temporal_layer_id = 0;
// Update LAST on layer 0, reference LAST.
ref_frame_config->refresh[0] = 1;
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
// Refresh lag_index slot, needed for lagging golen.
ref_frame_config->refresh[lag_index] = 1;
// Refresh GOLDEN every x base layer frames.
if (base_count % 32 == 0) ref_frame_config->refresh[3] = 1;
}
else {
layer_id->temporal_layer_id = 1;
// No updates on layer 1, reference LAST (TL0).
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
}
// Always reference golden and altref on TL0.
if (layer_id->temporal_layer_id == 0) {
ref_frame_config->reference[SVC_GOLDEN_FRAME] = 1;
ref_frame_config->reference[SVC_ALTREF_FRAME] = 1;
}
break;
case 2:
// 3-temporal layer:
// 1 3 5 7
// 2 6
// 0 4 8
if (superframe_cnt % 4 == 0) {
// Base layer.
layer_id->temporal_layer_id = 0;
// Update LAST on layer 0, reference LAST.
ref_frame_config->refresh[0] = 1;
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
}
else if ((superframe_cnt - 1) % 4 == 0) {
layer_id->temporal_layer_id = 2;
// First top layer: no updates, only reference LAST (TL0).
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
}
else if ((superframe_cnt - 2) % 4 == 0) {
layer_id->temporal_layer_id = 1;
// Middle layer (TL1): update LAST2, only reference LAST (TL0).
ref_frame_config->refresh[1] = 1;
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
}
else if ((superframe_cnt - 3) % 4 == 0) {
layer_id->temporal_layer_id = 2;
// Second top layer: no updates, only reference LAST.
// Set buffer idx for LAST to slot 1, since that was the slot
// updated in previous frame. So LAST is TL1 frame.
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 1;
ref_frame_config->ref_idx[SVC_LAST2_FRAME] = 0;
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
}
break;
case 3:
// 3 TL, same as above, except allow for predicting
// off 2 more references (GOLDEN and ALTREF), with
// GOLDEN updated periodically, and ALTREF lagging from
// LAST from ~4 frames. Both GOLDEN and ALTREF
// can only be updated on base temporal layer.
// Keep golden fixed at slot 3.
ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 3;
// Cyclically refresh slots 5, 6, 7, for lag altref.
lag_index = 5;
if (base_count > 0) {
lag_index = 5 + (base_count % 3);
if (superframe_cnt % 4 != 0) lag_index = 5 + ((base_count + 1) % 3);
}
// Set the altref slot to lag_index.
ref_frame_config->ref_idx[SVC_ALTREF_FRAME] = lag_index;
if (superframe_cnt % 4 == 0) {
// Base layer.
layer_id->temporal_layer_id = 0;
// Update LAST on layer 0, reference LAST.
ref_frame_config->refresh[0] = 1;
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
// Refresh GOLDEN every x ~10 base layer frames.
if (base_count % 10 == 0) ref_frame_config->refresh[3] = 1;
// Refresh lag_index slot, needed for lagging altref.
ref_frame_config->refresh[lag_index] = 1;
}
else if ((superframe_cnt - 1) % 4 == 0) {
layer_id->temporal_layer_id = 2;
// First top layer: no updates, only reference LAST (TL0).
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
}
else if ((superframe_cnt - 2) % 4 == 0) {
layer_id->temporal_layer_id = 1;
// Middle layer (TL1): update LAST2, only reference LAST (TL0).
ref_frame_config->refresh[1] = 1;
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
}
else if ((superframe_cnt - 3) % 4 == 0) {
layer_id->temporal_layer_id = 2;
// Second top layer: no updates, only reference LAST.
// Set buffer idx for LAST to slot 1, since that was the slot
// updated in previous frame. So LAST is TL1 frame.
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 1;
ref_frame_config->ref_idx[SVC_LAST2_FRAME] = 0;
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
}
// Every frame can reference GOLDEN AND ALTREF.
ref_frame_config->reference[SVC_GOLDEN_FRAME] = 1;
ref_frame_config->reference[SVC_ALTREF_FRAME] = 1;
// Allow for compound prediction for LAST-ALTREF and LAST-GOLDEN.
if (speed >= 7) {
ref_frame_comp_pred->use_comp_pred[2] = 1;
ref_frame_comp_pred->use_comp_pred[0] = 1;
}
break;
case 4:
// 3-temporal layer: but middle layer updates GF, so 2nd TL2 will
// only reference GF (not LAST). Other frames only reference LAST.
// 1 3 5 7
// 2 6
// 0 4 8
if (superframe_cnt % 4 == 0) {
// Base layer.
layer_id->temporal_layer_id = 0;
// Update LAST on layer 0, only reference LAST.
ref_frame_config->refresh[0] = 1;
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
}
else if ((superframe_cnt - 1) % 4 == 0) {
layer_id->temporal_layer_id = 2;
// First top layer: no updates, only reference LAST (TL0).
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
}
else if ((superframe_cnt - 2) % 4 == 0) {
layer_id->temporal_layer_id = 1;
// Middle layer (TL1): update GF, only reference LAST (TL0).
ref_frame_config->refresh[3] = 1;
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
}
else if ((superframe_cnt - 3) % 4 == 0) {
layer_id->temporal_layer_id = 2;
// Second top layer: no updates, only reference GF.
ref_frame_config->reference[SVC_GOLDEN_FRAME] = 1;
}
break;
case 5:
// 2 spatial layers, 1 temporal.
layer_id->temporal_layer_id = 0;
if (layer_id->spatial_layer_id == 0) {
// Reference LAST, update LAST.
ref_frame_config->refresh[0] = 1;
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
}
else if (layer_id->spatial_layer_id == 1) {
// Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1
// and GOLDEN to slot 0. Update slot 1 (LAST).
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 1;
ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 0;
ref_frame_config->refresh[1] = 1;
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
ref_frame_config->reference[SVC_GOLDEN_FRAME] = 1;
}
break;
case 6:
// 3 spatial layers, 1 temporal.
// Note for this case, we set the buffer idx for all references to be
// either LAST or GOLDEN, which are always valid references, since decoder
// will check if any of the 7 references is valid scale in
// valid_ref_frame_size().
layer_id->temporal_layer_id = 0;
if (layer_id->spatial_layer_id == 0) {
// Reference LAST, update LAST. Set all buffer_idx to 0.
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 0;
ref_frame_config->refresh[0] = 1;
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
}
else if (layer_id->spatial_layer_id == 1) {
// Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1
// and GOLDEN (and all other refs) to slot 0.
// Update slot 1 (LAST).
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 0;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 1;
ref_frame_config->refresh[1] = 1;
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
ref_frame_config->reference[SVC_GOLDEN_FRAME] = 1;
}
else if (layer_id->spatial_layer_id == 2) {
// Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 2
// and GOLDEN (and all other refs) to slot 1.
// Update slot 2 (LAST).
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 1;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 2;
ref_frame_config->refresh[2] = 1;
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
ref_frame_config->reference[SVC_GOLDEN_FRAME] = 1;
// For 3 spatial layer case: allow for top spatial layer to use
// additional temporal reference. Update every 10 frames.
if (enable_longterm_temporal_ref) {
ref_frame_config->ref_idx[SVC_ALTREF_FRAME] = REF_FRAMES - 1;
ref_frame_config->reference[SVC_ALTREF_FRAME] = 1;
if (base_count % 10 == 0)
ref_frame_config->refresh[REF_FRAMES - 1] = 1;
}
}
break;
case 7:
// 2 spatial and 3 temporal layer.
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
if (superframe_cnt % 4 == 0) {
// Base temporal layer
layer_id->temporal_layer_id = 0;
if (layer_id->spatial_layer_id == 0) {
// Reference LAST, update LAST
// Set all buffer_idx to 0
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 0;
ref_frame_config->refresh[0] = 1;
}
else if (layer_id->spatial_layer_id == 1) {
// Reference LAST and GOLDEN.
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 0;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 1;
ref_frame_config->refresh[1] = 1;
}
}
else if ((superframe_cnt - 1) % 4 == 0) {
// First top temporal enhancement layer.
layer_id->temporal_layer_id = 2;
if (layer_id->spatial_layer_id == 0) {
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 0;
ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 3;
ref_frame_config->refresh[3] = 1;
}
else if (layer_id->spatial_layer_id == 1) {
// Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1,
// GOLDEN (and all other refs) to slot 3.
// No update.
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 3;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 1;
}
}
else if ((superframe_cnt - 2) % 4 == 0) {
// Middle temporal enhancement layer.
layer_id->temporal_layer_id = 1;
if (layer_id->spatial_layer_id == 0) {
// Reference LAST.
// Set all buffer_idx to 0.
// Set GOLDEN to slot 5 and update slot 5.
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 0;
ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 5 - shift;
ref_frame_config->refresh[5 - shift] = 1;
}
else if (layer_id->spatial_layer_id == 1) {
// Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1,
// GOLDEN (and all other refs) to slot 5.
// Set LAST3 to slot 6 and update slot 6.
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 5 - shift;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 1;
ref_frame_config->ref_idx[SVC_LAST3_FRAME] = 6 - shift;
ref_frame_config->refresh[6 - shift] = 1;
}
}
else if ((superframe_cnt - 3) % 4 == 0) {
// Second top temporal enhancement layer.
layer_id->temporal_layer_id = 2;
if (layer_id->spatial_layer_id == 0) {
// Set LAST to slot 5 and reference LAST.
// Set GOLDEN to slot 3 and update slot 3.
// Set all other buffer_idx to 0.
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 0;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 5 - shift;
ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 3;
ref_frame_config->refresh[3] = 1;
}
else if (layer_id->spatial_layer_id == 1) {
// Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 6,
// GOLDEN to slot 3. No update.
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 0;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 6 - shift;
ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 3;
}
}
break;
case 8:
// 3 spatial and 3 temporal layer.
// Same as case 9 but overalap in the buffer slot updates.
// (shift = 2). The slots 3 and 4 updated by first TL2 are
// reused for update in TL1 superframe.
// Note for this case, frame order hint must be disabled for
// lower resolutios (operating points > 0) to be decoedable.
case 9:
// 3 spatial and 3 temporal layer.
// No overlap in buffer updates between TL2 and TL1.
// TL2 updates slot 3 and 4, TL1 updates 5, 6, 7.
// Set the references via the svc_ref_frame_config control.
// Always reference LAST.
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
if (superframe_cnt % 4 == 0) {
// Base temporal layer.
layer_id->temporal_layer_id = 0;
if (layer_id->spatial_layer_id == 0) {
// Reference LAST, update LAST.
// Set all buffer_idx to 0.
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 0;
ref_frame_config->refresh[0] = 1;
}
else if (layer_id->spatial_layer_id == 1) {
// Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1,
// GOLDEN (and all other refs) to slot 0.
// Update slot 1 (LAST).
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 0;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 1;
ref_frame_config->refresh[1] = 1;
}
else if (layer_id->spatial_layer_id == 2) {
// Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 2,
// GOLDEN (and all other refs) to slot 1.
// Update slot 2 (LAST).
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 1;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 2;
ref_frame_config->refresh[2] = 1;
}
}
else if ((superframe_cnt - 1) % 4 == 0) {
// First top temporal enhancement layer.
layer_id->temporal_layer_id = 2;
if (layer_id->spatial_layer_id == 0) {
// Reference LAST (slot 0).
// Set GOLDEN to slot 3 and update slot 3.
// Set all other buffer_idx to slot 0.
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 0;
ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 3;
ref_frame_config->refresh[3] = 1;
}
else if (layer_id->spatial_layer_id == 1) {
// Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1,
// GOLDEN (and all other refs) to slot 3.
// Set LAST2 to slot 4 and Update slot 4.
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 3;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 1;
ref_frame_config->ref_idx[SVC_LAST2_FRAME] = 4;
ref_frame_config->refresh[4] = 1;
}
else if (layer_id->spatial_layer_id == 2) {
// Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 2,
// GOLDEN (and all other refs) to slot 4.
// No update.
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 4;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 2;
}
}
else if ((superframe_cnt - 2) % 4 == 0) {
// Middle temporal enhancement layer.
layer_id->temporal_layer_id = 1;
if (layer_id->spatial_layer_id == 0) {
// Reference LAST.
// Set all buffer_idx to 0.
// Set GOLDEN to slot 5 and update slot 5.
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 0;
ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 5 - shift;
ref_frame_config->refresh[5 - shift] = 1;
}
else if (layer_id->spatial_layer_id == 1) {
// Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1,
// GOLDEN (and all other refs) to slot 5.
// Set LAST3 to slot 6 and update slot 6.
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 5 - shift;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 1;
ref_frame_config->ref_idx[SVC_LAST3_FRAME] = 6 - shift;
ref_frame_config->refresh[6 - shift] = 1;
}
else if (layer_id->spatial_layer_id == 2) {
// Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 2,
// GOLDEN (and all other refs) to slot 6.
// Set LAST3 to slot 7 and update slot 7.
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 6 - shift;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 2;
ref_frame_config->ref_idx[SVC_LAST3_FRAME] = 7 - shift;
ref_frame_config->refresh[7 - shift] = 1;
}
}
else if ((superframe_cnt - 3) % 4 == 0) {
// Second top temporal enhancement layer.
layer_id->temporal_layer_id = 2;
if (layer_id->spatial_layer_id == 0) {
// Set LAST to slot 5 and reference LAST.
// Set GOLDEN to slot 3 and update slot 3.
// Set all other buffer_idx to 0.
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 0;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 5 - shift;
ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 3;
ref_frame_config->refresh[3] = 1;
}
else if (layer_id->spatial_layer_id == 1) {
// Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 6,
// GOLDEN to slot 3. Set LAST2 to slot 4 and update slot 4.
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 0;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 6 - shift;
ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 3;
ref_frame_config->ref_idx[SVC_LAST2_FRAME] = 4;
ref_frame_config->refresh[4] = 1;
}
else if (layer_id->spatial_layer_id == 2) {
// Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 7,
// GOLDEN to slot 4. No update.
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 0;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 7 - shift;
ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 4;
}
}
break;
case 11:
// Simulcast mode for 3 spatial and 3 temporal layers.
// No inter-layer predicton, only prediction is temporal and single
// reference (LAST).
// No overlap in buffer slots between spatial layers. So for example,
// SL0 only uses slots 0 and 1.
// SL1 only uses slots 2 and 3.
// SL2 only uses slots 4 and 5.
// All 7 references for each inter-frame must only access buffer slots
// for that spatial layer.
// On key (super)frames: SL1 and SL2 must have no references set
// and must refresh all the slots for that layer only (so 2 and 3
// for SL1, 4 and 5 for SL2). The base SL0 will be labelled internally
// as a Key frame (refresh all slots). SL1/SL2 will be labelled
// internally as Intra-only frames that allow that stream to be decoded.
// These conditions will allow for each spatial stream to be
// independently decodeable.
// Initialize all references to 0 (don't use reference).
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->reference[i] = 0;
// Initialize as no refresh/update for all slots.
for (i = 0; i < REF_FRAMES; i++) ref_frame_config->refresh[i] = 0;
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 0;
if (is_key_frame) {
if (layer_id->spatial_layer_id == 0) {
// Assign LAST/GOLDEN to slot 0/1.
// Refesh slots 0 and 1 for SL0.
// SL0: this will get set to KEY frame internally.
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 0;
ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 1;
ref_frame_config->refresh[0] = 1;
ref_frame_config->refresh[1] = 1;
}
else if (layer_id->spatial_layer_id == 1) {
// Assign LAST/GOLDEN to slot 2/3.
// Refesh slots 2 and 3 for SL1.
// This will get set to Intra-only frame internally.
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 2;
ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 3;
ref_frame_config->refresh[2] = 1;
ref_frame_config->refresh[3] = 1;
}
else if (layer_id->spatial_layer_id == 2) {
// Assign LAST/GOLDEN to slot 4/5.
// Refresh slots 4 and 5 for SL2.
// This will get set to Intra-only frame internally.
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 4;
ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 5;
ref_frame_config->refresh[4] = 1;
ref_frame_config->refresh[5] = 1;
}
}
else if (superframe_cnt % 4 == 0) {
// Base temporal layer: TL0
layer_id->temporal_layer_id = 0;
if (layer_id->spatial_layer_id == 0) {
// SL0
// Reference LAST. Assign all references to either slot
// 0 or 1. Here we assign LAST to slot 0, all others to 1.
// Update slot 0 (LAST).
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 1;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 0;
ref_frame_config->refresh[0] = 1;
}
else if (layer_id->spatial_layer_id == 1) {
// SL1
// Reference LAST. Assign all references to either slot
// 2 or 3. Here we assign LAST to slot 2, all others to 3.
// Update slot 2 (LAST).
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 3;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 2;
ref_frame_config->refresh[2] = 1;
}
else if (layer_id->spatial_layer_id == 2) {
// SL2
// Reference LAST. Assign all references to either slot
// 4 or 5. Here we assign LAST to slot 4, all others to 5.
// Update slot 4 (LAST).
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 5;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 4;
ref_frame_config->refresh[4] = 1;
}
}
else if ((superframe_cnt - 1) % 4 == 0) {
// First top temporal enhancement layer: TL2
layer_id->temporal_layer_id = 2;
if (layer_id->spatial_layer_id == 0) {
// SL0
// Reference LAST (slot 0). Assign other references to slot 1.
// No update/refresh on any slots.
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 1;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 0;
}
else if (layer_id->spatial_layer_id == 1) {
// SL1
// Reference LAST (slot 2). Assign other references to slot 3.
// No update/refresh on any slots.
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 3;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 2;
}
else if (layer_id->spatial_layer_id == 2) {
// SL2
// Reference LAST (slot 4). Assign other references to slot 4.
// No update/refresh on any slots.
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 5;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 4;
}
}
else if ((superframe_cnt - 2) % 4 == 0) {
// Middle temporal enhancement layer: TL1
layer_id->temporal_layer_id = 1;
if (layer_id->spatial_layer_id == 0) {
// SL0
// Reference LAST (slot 0).
// Set GOLDEN to slot 1 and update slot 1.
// This will be used as reference for next TL2.
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 1;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 0;
ref_frame_config->refresh[1] = 1;
}
else if (layer_id->spatial_layer_id == 1) {
// SL1
// Reference LAST (slot 2).
// Set GOLDEN to slot 3 and update slot 3.
// This will be used as reference for next TL2.
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 3;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 2;
ref_frame_config->refresh[3] = 1;
}
else if (layer_id->spatial_layer_id == 2) {
// SL2
// Reference LAST (slot 4).
// Set GOLDEN to slot 5 and update slot 5.
// This will be used as reference for next TL2.
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 5;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 4;
ref_frame_config->refresh[5] = 1;
}
}
else if ((superframe_cnt - 3) % 4 == 0) {
// Second top temporal enhancement layer: TL2
layer_id->temporal_layer_id = 2;
if (layer_id->spatial_layer_id == 0) {
// SL0
// Reference LAST (slot 1). Assign other references to slot 0.
// No update/refresh on any slots.
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 0;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 1;
}
else if (layer_id->spatial_layer_id == 1) {
// SL1
// Reference LAST (slot 3). Assign other references to slot 2.
// No update/refresh on any slots.
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 2;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 3;
}
else if (layer_id->spatial_layer_id == 2) {
// SL2
// Reference LAST (slot 5). Assign other references to slot 4.
// No update/refresh on any slots.
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
for (i = 0; i < INTER_REFS_PER_FRAME; i++)
ref_frame_config->ref_idx[i] = 4;
ref_frame_config->ref_idx[SVC_LAST_FRAME] = 5;
}
}
if (!simulcast_mode && layer_id->spatial_layer_id > 0) {
// Always reference GOLDEN (inter-layer prediction).
ref_frame_config->reference[SVC_GOLDEN_FRAME] = 1;
if (ksvc_mode) {
// KSVC: only keep the inter-layer reference (GOLDEN) for
// superframes whose base is key.
if (!is_key_frame) ref_frame_config->reference[SVC_GOLDEN_FRAME] = 0;
}
if (is_key_frame && layer_id->spatial_layer_id > 1) {
// On superframes whose base is key: remove LAST to avoid prediction
// off layer two levels below.
ref_frame_config->reference[SVC_LAST_FRAME] = 0;
}
}
// For 3 spatial layer case 8 (where there is free buffer slot):
// allow for top spatial layer to use additional temporal reference.
// Additional reference is only updated on base temporal layer, every
// 10 TL0 frames here.
if (!simulcast_mode && enable_longterm_temporal_ref &&
layer_id->spatial_layer_id == 2 && layering_mode == 8) {
ref_frame_config->ref_idx[SVC_ALTREF_FRAME] = REF_FRAMES - 1;
if (!is_key_frame) ref_frame_config->reference[SVC_ALTREF_FRAME] = 1;
if (base_count % 10 == 0 && layer_id->temporal_layer_id == 0)
ref_frame_config->refresh[REF_FRAMES - 1] = 1;
}
break;
default: assert(0); die(
"Error: Unsupported temporal layering mode!\n");
}
}
static void write_literal(
struct aom_write_bit_buffer *wb, uint32_t data,
uint8_t bits, uint32_t offset = 0) {
if (bits > 32) {
die(
"Invalid bits value %d > 32\n", bits);
}
const uint32_t max =
static_cast<uint32_t>(((uint64_t)1 << bits) - 1);
if (data < offset || (data - offset) > max) {
die(
"Invalid data, value %u out of range [%u, %" PRIu64
"]\n", data, offset,
(uint64_t)max + offset);
}
aom_wb_write_unsigned_literal(wb, data - offset, bits);
}
static void write_depth_representation_element(
struct aom_write_bit_buffer *buffer,
const std::pair<libaom_examples::DepthRepresentationElement,
bool>
&element) {
if (!element.second) {
return;
}
write_literal(buffer, element.first.sign_flag, 1);
write_literal(buffer, element.first.exponent, 7);
if (element.first.mantissa_len == 0 || element.first.mantissa_len > 32) {
die(
"Invalid mantissan_len %d\n", element.first.mantissa_len);
}
write_literal(buffer, element.first.mantissa_len - 1, 5);
write_literal(buffer, element.first.mantissa, element.first.mantissa_len);
}
static void write_color_properties(
struct aom_write_bit_buffer *buffer,
const std::pair<libaom_examples::ColorProperties,
bool> &color_properties) {
write_literal(buffer, color_properties.second, 1);
if (color_properties.second) {
write_literal(buffer, color_properties.first.color_range, 1);
write_literal(buffer, color_properties.first.color_primaries, 8);
write_literal(buffer, color_properties.first.transfer_characteristics, 8);
write_literal(buffer, color_properties.first.matrix_coefficients, 8);
}
else {
write_literal(buffer, 0, 1);
// reserved_1bit
}
}
static void add_multilayer_metadata(
aom_image_t *frame,
const libaom_examples::MultilayerMetadata &multilayer) {
// Pretty large buffer to accommodate the largest multilayer metadata
// possible, with 4 alpha segmentation layers (each can be up to about 66kB).
std::vector<uint8_t> data(66000 * multilayer.layers.size());
struct aom_write_bit_buffer buffer = { data.data(), 0 };
write_literal(&buffer, multilayer.use_case, 6);
if (multilayer.layers.empty()) {
die(
"Invalid multilayer metadata, no layers found\n");
}
else if (multilayer.layers.size() > MAX_NUM_SPATIAL_LAYERS) {
die(
"Invalid multilayer metadata, too many layers (max is %d)\n",
MAX_NUM_SPATIAL_LAYERS);
}
write_literal(&buffer, (
int)multilayer.layers.size() - 1, 2);
assert(buffer.bit_offset % 8 == 0);
for (size_t i = 0; i < multilayer.layers.size(); ++i) {
const libaom_examples::LayerMetadata &layer = multilayer.layers[i];
// Alpha info with segmentation with labels can be up to about 66k bytes,
// which requires 3 bytes to encode in leb128.
const int bytes_reserved_for_size = 3;
// Placeholder for layer_metadata_size which will be written later.
write_literal(&buffer, 0, bytes_reserved_for_size * 8);
const uint32_t metadata_start = buffer.bit_offset;
write_literal(&buffer, (
int)i, 2);
// ml_spatial_id
write_literal(&buffer, layer.layer_type, 5);
write_literal(&buffer, layer.luma_plane_only_flag, 1);
write_literal(&buffer, layer.layer_view_type, 3);
write_literal(&buffer, layer.group_id, 2);
write_literal(&buffer, layer.layer_dependency_idc, 3);
write_literal(&buffer, layer.layer_metadata_scope, 2);
write_literal(&buffer, 0, 4);
// ml_reserved_4bits
if (i > 0) {
write_color_properties(&buffer, layer.layer_color_description);
}
else {
write_literal(&buffer, 0, 2);
// ml_reserved_2bits
}
assert(buffer.bit_offset % 8 == 0);
if (layer.layer_type == libaom_examples::MULTILAYER_LAYER_TYPE_ALPHA &&
layer.layer_metadata_scope >= libaom_examples::SCOPE_GLOBAL) {
const libaom_examples::AlphaInformation &alpha_info =
layer.global_alpha_info;
write_literal(&buffer, alpha_info.alpha_use_idc, 3);
write_literal(&buffer, alpha_info.alpha_bit_depth, 3,
/*offset=*/8);
write_literal(&buffer, alpha_info.alpha_clip_idc, 2);
write_literal(&buffer, alpha_info.alpha_incr_flag, 1);
write_literal(&buffer, alpha_info.alpha_transparent_value,
alpha_info.alpha_bit_depth + 1);
write_literal(&buffer, alpha_info.alpha_opaque_value,
alpha_info.alpha_bit_depth + 1);
if (buffer.bit_offset % 8 != 0) {
// ai_byte_alignment_bits
write_literal(&buffer, 0, 8 - (buffer.bit_offset % 8));
}
assert(buffer.bit_offset % 8 == 0);
if (alpha_info.alpha_use_idc == libaom_examples::ALPHA_STRAIGHT) {
write_literal(&buffer, 0, 6);
// ai_reserved_6bits
write_color_properties(&buffer, alpha_info.alpha_color_description);
}
else if (alpha_info.alpha_use_idc ==
libaom_examples::ALPHA_SEGMENTATION) {
write_literal(&buffer, 0, 7);
// ai_reserved_7bits
write_literal(&buffer, !alpha_info.label_type_id.empty(), 1);
if (!alpha_info.label_type_id.empty()) {
const size_t num_values =
std::abs(alpha_info.alpha_transparent_value -
alpha_info.alpha_opaque_value) +
1;
if (!alpha_info.label_type_id.empty() &&
alpha_info.label_type_id.size() != num_values) {
die(
"Invalid multilayer metadata, label_type_id size must be "
"equal to the range of alpha values between "
"alpha_transparent_value and alpha_opaque_value (expected "
"%d values, found %d values)\n",
(
int)num_values, (
int)alpha_info.label_type_id.size());
}
for (size_t j = 0; j < num_values; ++j) {
write_literal(&buffer, alpha_info.label_type_id[j], 16);
}
}
}
assert(buffer.bit_offset % 8 == 0);
--> --------------------
--> maximum size reached
--> --------------------
