Invalid data when creating mkv container with h264 stream because extradata is null

Question

I need to encode raw frames into an h264 stream inside an mkv container using ffmpeg's c api. All the examples I could find copy existing codec parameters from a decoder so I had to modify it a bit.

#include <cstdio>
#include <cstring>
#include <iostream>

extern "C" {
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
#include <libavutil/imgutils.h>
#include <libavutil/opt.h>
#include <libswscale/swscale.h>
}

static char error_msg[4096] = "";


int main(int argc, char **argv) {
  int err;
  char *filename = (char *)av_malloc(4096);
  AVCodec *codec = nullptr;
  SwsContext *color_converter = nullptr;
  AVCodecContext *encoder = nullptr;
  AVFormatContext *container = nullptr;
  AVStream *stream = nullptr;

  std::snprintf(filename, sizeof(filename), "%s", "out.mkv");

  // Create encoder
  codec = avcodec_find_encoder_by_name("libx264");
  if (codec == NULL) {
    std::cerr << "codec not found" << std::endl;
    goto cleanup;
  }

  encoder = avcodec_alloc_context3(codec);
  if (encoder == nullptr) {
    std::cerr << "failed to allocate encoder" << std::endl;
    goto cleanup;
  }

  // Configure encoder
  encoder->color_range = AVCOL_RANGE_JPEG;
  encoder->height = 1024;
  encoder->width = 1280;
  encoder->sample_aspect_ratio = {1, 1};
  encoder->pix_fmt = AV_PIX_FMT_YUVJ420P;
  encoder->time_base = {1, 10};
  encoder->framerate = {10, 1};

  err = av_opt_set_double(encoder->priv_data, "crf", 0.0, 0);
  if (err < 0) {
    av_make_error_string(error_msg, sizeof(error_msg), err);
    std::cerr << "failed to initialize encoder: " << error_msg << std::endl;
    goto cleanup;
  }

  err = avcodec_open2(encoder, codec, nullptr);
  if (err < 0) {
    av_make_error_string(error_msg, sizeof(error_msg), err);
    std::cerr << "failed to initialize encoder: " << error_msg << std::endl;
    goto cleanup;
  }

  // Create container
  avformat_alloc_output_context2(&container, NULL, NULL, filename);
  if (container == NULL) {
    std::cerr << "failed to allocate container" << std::endl;
    goto cleanup;
  }

  // Add stream
  stream = avformat_new_stream(container, nullptr);
  if (stream == NULL) {
    std::cerr << "failed to create new stream" << std::endl;
    goto cleanup;
  }

  stream->time_base = encoder->time_base;
  err = avcodec_parameters_from_context(stream->codecpar, encoder);
  if (err < 0) {
    av_make_error_string(error_msg, sizeof(error_msg), err);
    std::cerr << "failed to configure stream: " << error_msg << std::endl;
    goto cleanup;
  }

  // encoder->extradata_size == 0 at this point!

  // Open file and write file header
  err = avio_open(&container->pb, filename, AVIO_FLAG_WRITE);
  if (err < 0) {
    av_make_error_string(error_msg, sizeof(error_msg), err);
    std::cerr << "failed to open output file: " << error_msg << std::endl;
    goto cleanup;
  }

  err = avformat_write_header(container, nullptr);
  // fails in ff_isom_write_avcc because stream->codecpar->extradata is null.
  if (err < 0) {
    av_make_error_string(error_msg, sizeof(error_msg), err);
    std::cerr << "failed to set-up container: " << error_msg << std::endl;
    goto cleanup;
  }

cleanup:
  avcodec_close(encoder);
  if (container != nullptr) {
    avio_closep(&container->pb);
  }
  avcodec_free_context(&encoder);
  sws_freeContext(color_converter);
  avformat_free_context(container);
  av_free(filename);

  return 0;
}

This fails with EINVAL (Invalid data found when processing input) at the avformat_write_header line (more precisely in ff_isom_write_avcc inside the ffmpeg library) because stream->codecpar->extradata is null. The same code works for an mp4 container and I have no clue how to initialize stream->codecpar->extradata (or encoder->extradata) manually.

I have checked similar questions about that problem but could not find a conclusive answer:

• https://stackoverflow.com/a/22231863/5786475 says extradata can be null when h264 is in annex b format
• https://stackoverflow.com/a/27471912/5786475 says it shouldn't be null.

Answer 1

Set AV_CODEC_FLAG_GLOBAL_HEADER for encoder->flags before opening. This will tell x264 to add extradata to the encoder context instead of sending it as in-band within the encoded bitstream. The Matroska muxer only looks at AVStream codec parameters for this extradata and it is copied there from encoder context and not any packets in your code.