Improve error handling

c_src/decoder.c

@@ -41,7 +41,8 @@ int decoder_alloc(Decoder **ctx, DecoderOpts opts) {
   int errn;
 
   codec = avcodec_find_decoder((enum AVCodecID)opts.codec_id);
-  codec_ctx = avcodec_alloc_context3(codec);
+  if (!(codec_ctx = avcodec_alloc_context3(codec)))
+    return AVERROR(ENOMEM);
 
   if ((errn = avcodec_parameters_to_context(codec_ctx, opts.params)) < 0)
     return errn;
@@ -51,9 +52,13 @@ int decoder_alloc(Decoder **ctx, DecoderOpts opts) {
 
   codec_ctx->pkt_timebase = opts.timebase;
 
-  ictx = (Decoder *)malloc(sizeof(Decoder));
+  if (!(ictx = (Decoder *)malloc(sizeof(Decoder))))
+    return AVERROR(ENOMEM);
+
   ictx->codec_ctx = codec_ctx;
-  ictx->output.ch_layout = malloc(sizeof(AVChannelLayout));
+  if (!(ictx->output.ch_layout = malloc(sizeof(AVChannelLayout))))
+    return AVERROR(ENOMEM);
+
   av_channel_layout_copy(ictx->output.ch_layout, &codec_ctx->ch_layout);
   ictx->output.ch_layout->nb_channels = opts.output.nb_channels;
   ictx->output.sample_rate = opts.output.sample_rate;
@@ -83,7 +88,9 @@ int decoder_read_frame(Decoder *ctx, AVFrame *frame) {
     int next_pts = swr_next_pts(ctx->resampler_ctx, frame->pts);
 
     AVFrame *resampled_frame;
-    resampled_frame = av_frame_alloc();
+    if (!(resampled_frame = av_frame_alloc()))
+      return AVERROR(ENOMEM);
+
     resampled_frame->nb_samples = frame->nb_samples;
     resampled_frame->ch_layout = *ctx->output.ch_layout;
     resampled_frame->sample_rate = ctx->output.sample_rate;

c_src/demuxer.c

@@ -7,14 +7,18 @@ int demuxer_alloc_from_file(Demuxer **ctx, char *path) {
   Demuxer *demuxer;
   int errn;
 
-  fmt_ctx = avformat_alloc_context();
+  if (!(fmt_ctx = avformat_alloc_context()))
+    return AVERROR(ENOMEM);
+
   if ((errn = avformat_open_input(&fmt_ctx, path, NULL, NULL)) < 0)
     return errn;
 
   if ((errn = avformat_find_stream_info(fmt_ctx, NULL)) < 0)
     goto fail;
 
-  demuxer = (Demuxer *)malloc(sizeof(Demuxer));
+  if (!(demuxer = (Demuxer *)malloc(sizeof(Demuxer))))
+    return AVERROR(ENOMEM);
+
   demuxer->fmt_ctx = fmt_ctx;
   *ctx = demuxer;
   return 0;

c_src/libav.c

@@ -1,3 +1,4 @@
+#include "libavutil/frame.h"
 #include "libavutil/samplefmt.h"
 #include <decoder.h>
 #include <demuxer.h>
@@ -86,7 +87,8 @@ ERL_NIF_TERM enif_demuxer_read_packet(ErlNifEnv *env, int argc,
   AVPacket *packet;
   int ret;
 
-  packet = av_packet_alloc();
+  if (!(packet = av_packet_alloc()))
+    return enif_make_av_error(env, AVERROR(ENOMEM));
 
   enif_get_demuxer(env, argv[0], &ctx);
 
@@ -344,16 +346,20 @@ ERL_NIF_TERM enif_decoder_add_data(ErlNifEnv *env, int argc,
     return enif_make_av_error(env, errn);
 
   list = enif_make_list(env, 0);
-  frame = av_frame_alloc();
+  if (!(frame = av_frame_alloc()))
+    return enif_make_av_error(env, AVERROR(ENOMEM));
+
   while ((errn = decoder_read_frame(ctx, frame)) == 0) {
-    AVFrame *oframe;
-    oframe = av_frame_alloc();
-    av_frame_ref(oframe, frame);
+    int errn;
+    AVFrame **frame_res;
 
     // Make the resource take ownership on the context.
-    AVFrame **frame_res =
-        enif_alloc_resource(FRAME_RES_TYPE, sizeof(AVFrame *));
-    *frame_res = oframe;
+    if (!(frame_res = enif_alloc_resource(FRAME_RES_TYPE, sizeof(AVFrame *))))
+      return enif_make_av_error(env, AVERROR(ENOMEM));
+
+    *frame_res = av_frame_alloc();
+    if ((errn = av_frame_ref(*frame_res, frame) < 0))
+      return enif_make_av_error(env, errn);
 
     ERL_NIF_TERM term = enif_make_resource(env, frame_res);