ijkplayer是一个基于FFmpeg的轻量级Android/iOS视频播放器。ijkplayer默认不支持截图和录制功能。要想实现截图、录制功能就需要另外开发来实现。
本文的录制实现需要ijkplayer编译增加编码器。ijkplayer编译支持x264编码器可参考：
https://www.jianshu.com/p/765a71679b2a
精简的工程目录如下：
工程简要目录结构.png
android工程改成了cmake模式，方便编译、开发、调试，目录结构如下：
工程目录结构.png
ijkplayer-java模块修改
IjkMediaPlayer.java文件增加如下代码：
public native boolean getCurrentFrame(Bitmap bitmap);
public native int startRecord(String recordVideoPath);
public native int stopRecord();
@Override
public boolean doCapture(Bitmap bitmap) {
return getCurrentFrame(bitmap);
}
@Override
public boolean doStartRecord(String recordVideoPath) {
return (startRecord(recordVideoPath)==0)?true:false;
}
@Override
public void doStopRecord() {
stopRecord();
}
主要是jni在java层对应的方法以及对外开放的调用方法。
native-lib模块修改
ijkplayer_jni.c文件修改
static jboolean
IjkMediaPlayer_getCurrentFrame(JNIEnv *env, jobject thiz, jobject bitmap)
{
jboolean retval = JNI_TRUE;
IjkMediaPlayer *mp = jni_get_media_player(env, thiz);
JNI_CHECK_GOTO(mp, env, NULL, "mpjni: getCurrentFrame: null mp", LABEL_RETURN);
uint8_t *frame_buffer = NULL;
if (0 > AndroidBitmap_lockPixels(env, bitmap, (void **)&frame_buffer)) {
(*env)->ThrowNew(env, "java/io/IOException", "Unable to lock pixels.");
return JNI_FALSE;
}
ijkmp_get_current_frame(mp, frame_buffer);
if (0 > AndroidBitmap_unlockPixels(env, bitmap)) {
(*env)->ThrowNew(env, "java/io/IOException", "Unable to unlock pixels.");
return JNI_FALSE;
}
LABEL_RETURN:
ijkmp_dec_ref_p(&mp);
return retval;
}
static jint IjkMediaPlayer_startRecord(JNIEnv* env,jobject thiz,jstring recordFilePath){
IjkMediaPlayer *mp = jni_get_media_player(env, thiz);
const char *c_record_path = NULL;
c_record_path = (*env)->GetStringUTFChars(env, recordFilePath, NULL );
if(mp->ffplayer->dx_recordRelData.isOnEncoding == DX_RECORD_ENCODING_OFF){
ijkmp_start_record(mp,c_record_path);
}
return mp->ffplayer->dx_recordRelData.isOnEncoding;
}
static jint IjkMediaPlayer_stopRecord(JNIEnv* env,jobject thiz){
IjkMediaPlayer *mp = jni_get_media_player(env, thiz);
ijkmp_stop_record(mp);
return 0;
}
static JNINativeMethod g_methods[] = {
···
{ "_setFrameAtTime", "(Ljava/lang/String;JJII)V", (void *) IjkMediaPlayer_setFrameAtTime },
{ "getCurrentFrame", "(Landroid/graphics/Bitmap;)Z", (void *) IjkMediaPlayer_getCurrentFrame },
{ "startRecord", "(Ljava/lang/String;)I", (void *) IjkMediaPlayer_startRecord },
{ "stopRecord", "()I", (void *) IjkMediaPlayer_stopRecord },
};
增加如上3个函数，并与java层对应的实现函数进行动态注册关联。
ijkplayer.h文件修改
void ijkmp_get_current_frame(IjkMediaPlayer *mp, uint8_t *frame_buf);
void ijkmp_start_record(IjkMediaPlayer *mp, const char *recordFileName);
void ijkmp_stop_record(IjkMediaPlayer *mp);
ijkplayer.c文件修改
void ijkmp_get_current_frame(IjkMediaPlayer *mp, uint8_t *frame_buf)
{
assert(mp);
pthread_mutex_lock(&mp->mutex);
ijkmp_get_current_frame_l(mp, frame_buf);
pthread_mutex_unlock(&mp->mutex);
}
void ijkmp_start_record(IjkMediaPlayer *mp, const char *recordFileName){
MPTRACE("ijkmp_start_record: dj prepare to start record %s\n",recordFileName);
ffp_start_record(mp->ffplayer,recordFileName);
}
void ijkmp_stop_record(IjkMediaPlayer *mp){
MPTRACE("ijkmp_stop_record: dj stop record\n");
ffp_stop_record(mp->ffplayer);
}
ff_fplay.h文件修改如下：
void ffp_get_current_frame_l(FFPlayer *ffp, uint8_t *frame_buf);
void ffp_start_record(FFPlayer *ffp, const char *file_name);
void ffp_stop_record(FFPlayer *ffp);
ff_ffplay.c文件修改如下：
void ffp_get_current_frame_l(FFPlayer *ffp, uint8_t *frame_buf)
{
ALOGD("=============>start snapshot\n");
VideoState *is = ffp->is;
Frame *vp;
int i = 0, linesize = 0, pixels = 0;
uint8_t *src;
vp = &is->pictq.queue[is->pictq.rindex];
int height = vp->bmp->h;
int width = vp->bmp->w;
ALOGD("=============>%d X %d === %d\n", width, height, vp->bmp->pitches[0]);
// copy data to bitmap in java code
linesize = vp->bmp->pitches[0];
src = vp->bmp->pixels[0];
pixels = width * 4;
for (i = 0; i < height; i++) {
memcpy(frame_buf + i * pixels, src + i * linesize, pixels);
}
ALOGD("=============>end snapshot\n");
}
void ffp_start_record(FFPlayer *ffp, const char *file_name)
{
ffp->dx_recordRelData.windex = 0;
ffp->dx_recordRelData.rindex = 0;
ffp->dx_recordRelData.fileName = file_name;
// ffp->dx_recordRelData.isInRecord = DX_RECORD_STATUS_ON;
ALOGD("ffp_start_record filename=%s recordStatus=%d\n",file_name,ffp->dx_recordRelData.isInRecord);
ffp->dx_recordRelData.isInRecord = DX_RECORD_STATUS_ON;
pthread_create(&(ffp->dx_recordRelData.recThreadid),NULL,doRecordFile,(void *)(&(ffp->dx_recordRelData)));
}
void ffp_stop_record(FFPlayer *ffp){
ALOGD("ffp_stop_record filename=%s recordStatus=%d\n",ffp->dx_recordRelData.fileName,ffp->dx_recordRelData.isInRecord);
ffp->dx_recordRelData.isInRecord = DX_RECORD_STATUS_OFF;
}
//解码后将解码数据保存下来
static int decoder_decode_frame(FFPlayer *ffp, Decoder *d, AVFrame *frame, AVSubtitle *sub) {
int ret = AVERROR(EAGAIN);
for (;;) {
AVPacket pkt;
if (d->queue->serial == d->pkt_serial) {
do {
if (d->queue->abort_request)
return -1;
switch (d->avctx->codec_type) {
case AVMEDIA_TYPE_VIDEO:
ret = avcodec_receive_frame(d->avctx, frame);
if (ret >= 0) {
ffp->stat.vdps = SDL_SpeedSamplerAdd(&ffp->vdps_sampler, FFP_SHOW_VDPS_AVCODEC, "vdps[avcodec]");
if (ffp->decoder_reorder_pts == -1) {
frame->pts = frame->best_effort_timestamp;
} else if (!ffp->decoder_reorder_pts) {
frame->pts = frame->pkt_dts;
}
//mody by dj add video record process
if (frame->format == AV_PIX_FMT_YUV420P && ffp->dx_recordRelData.isInRecord == DX_RECORD_STATUS_ON){
if (frame->width >0 && frame->height >0){
DX_FrameData frData;
frData.data0 = (uint8_t *)malloc((size_t)frame->linesize[0] * frame->height);
frData.data1 = (uint8_t *)malloc((size_t)frame->linesize[1]*frame->height/2);
frData.data2 = (uint8_t *)malloc((size_t)frame->linesize[1]*frame->height/2);
frData.dataNum = 3;
frData.frameType = DX_FRAME_TYPE_VIDEO;
frData.lineSize0 = frame->linesize[0];
frData.lineSize1 = frame->linesize[1];
frData.lineSize2 = frame->linesize[2];
frData.format = frame->format;
memcpy(frData.data0,frame->data[0],frame->linesize[0]*frame->height);
memcpy(frData.data1,frame->data[1],frame->linesize[1]*frame->height/2);
memcpy(frData.data2,frame->data[2],frame->linesize[2]*frame->height/2);
int windex = ffp->dx_recordRelData.windex;
ffp->dx_recordRelData.recordFramesQueue[windex] = frData;
ffp->dx_recordRelData.windex += 1;
ffp->dx_recordRelData.srcFormat.height = frame->height;
ffp->dx_recordRelData.srcFormat.width = frame->width;
}
}
}
break;
case AVMEDIA_TYPE_AUDIO:
ret = avcodec_receive_frame(d->avctx, frame);
if (ret >= 0) {
AVRational tb = (AVRational){1, frame->sample_rate};
if (frame->pts != AV_NOPTS_VALUE)
frame->pts = av_rescale_q(frame->pts, av_codec_get_pkt_timebase(d->avctx), tb);
else if (d->next_pts != AV_NOPTS_VALUE)
frame->pts = av_rescale_q(d->next_pts, d->next_pts_tb, tb);
if (frame->pts != AV_NOPTS_VALUE) {
d->next_pts = frame->pts + frame->nb_samples;
d->next_pts_tb = tb;
}
//mody by dj add video record process
if(ffp->dx_recordRelData.isInRecord == DX_RECORD_STATUS_ON && frame->format == AV_SAMPLE_FMT_FLTP){
if(frame->linesize[0] >0){
DX_FrameData frData;
frData.data0 = (uint8_t *)av_malloc(frame->linesize[0]);
memcpy(frData.data0,frame->data[0],frame->linesize[0]);
frData.data1 = (uint8_t *)av_malloc(frame->linesize[0]);
memcpy(frData.data1,frame->data[1],frame->linesize[0]);
frData.frameType = DX_FRAME_TYPE_AUDIO;
frData.dataNum = 2;
frData.nb_samples = frame->nb_samples;
frData.channel_layout = frame->channel_layout;
frData.channels = frame->channels;
frData.lineSize0 = frame->linesize[0];
frData.lineSize1 = frame->linesize[0];
frData.format = frame->format;
int windex = ffp->dx_recordRelData.windex;
ffp->dx_recordRelData.recordFramesQueue[windex] = frData;
ffp->dx_recordRelData.windex += 1;
}
}
// else if(ffp->dx_recordRelData.isInRecord == DX_RECORD_STATUS_ON && frame->format == AV_SAMPLE_FMT_S16){
// if(frame->linesize[0] >0){
// DX_FrameData frData;
// frData.data0 = (uint8_t *)av_malloc(frame->linesize[0]);
// memcpy(frData.data0,frame->data[0],frame->linesize[0]);
// frData.frameType = DX_FRAME_TYPE_AUDIO;
// frData.dataNum = 1;
// frData.nb_samples = frame->nb_samples;
// frData.channel_layout = frame->channel_layout;
// frData.channels = frame->channels;
// frData.lineSize0 = frame->linesize[0];
// frData.format = frame->format;
// int windex = ffp->dx_recordRelData.windex;
// ffp->dx_recordRelData.recordFramesQueue[windex] = frData;
// ffp->dx_recordRelData.windex += 1;
// }
// }
}
break;
default:
break;
}
if (ret == AVERROR_EOF) {
d->finished = d->pkt_serial;
avcodec_flush_buffers(d->avctx);
return 0;
}
if (ret >= 0)
return 1;
} while (ret != AVERROR(EAGAIN));
}
do {
if (d->queue->nb_packets == 0)
SDL_CondSignal(d->empty_queue_cond);
if (d->packet_pending) {
av_packet_move_ref(&pkt, &d->pkt);
d->packet_pending = 0;
} else {
if (packet_queue_get_or_buffering(ffp, d->queue, &pkt, &d->pkt_serial, &d->finished) < 0)
return -1;
}
} while (d->queue->serial != d->pkt_serial);
if (pkt.data == flush_pkt.data) {
avcodec_flush_buffers(d->avctx);
d->finished = 0;
d->next_pts = d->start_pts;
d->next_pts_tb = d->start_pts_tb;
} else {
if (d->avctx->codec_type == AVMEDIA_TYPE_SUBTITLE) {
int got_frame = 0;
ret = avcodec_decode_subtitle2(d->avctx, sub, &got_frame, &pkt);
if (ret < 0) {
ret = AVERROR(EAGAIN);
} else {
if (got_frame && !pkt.data) {
d->packet_pending = 1;
av_packet_move_ref(&d->pkt, &pkt);
}
ret = got_frame ? 0 : (pkt.data ? AVERROR(EAGAIN) : AVERROR_EOF);
}
} else {
if (avcodec_send_packet(d->avctx, &pkt) == AVERROR(EAGAIN)) {
av_log(d->avctx, AV_LOG_ERROR, "Receive_frame and send_packet both returned EAGAIN, which is an API violation.\n");
d->packet_pending = 1;
av_packet_move_ref(&d->pkt, &pkt);
}
}
av_packet_unref(&pkt);
}
}
}
在ff_ffplay_def.h文件中将dj_record.h头文件include进来
#include "ijkmeta.h"
#include "dj_record.h"
typedef struct FFPlayer {
const AVClass *av_class;
/* ffplay context */
VideoState *is;
/* format/codec options */
AVDictionary *format_opts;
AVDictionary *codec_opts;
AVDictionary *sws_dict;
AVDictionary *player_opts;
AVDictionary *swr_opts;
AVDictionary *swr_preset_opts;
/* ffplay options specified by the user */
#ifdef FFP_MERGE
AVInputFormat *file_iformat;
#endif
char *input_filename;
#ifdef FFP_MERGE
const char *window_title;
int fs_screen_width;
int fs_screen_height;
int default_width;
int default_height;
int screen_width;
int screen_height;
#endif
int audio_disable;
int video_disable;
int subtitle_disable;
const char* wanted_stream_spec[AVMEDIA_TYPE_NB];
int seek_by_bytes;
int display_disable;
int show_status;
int av_sync_type;
int64_t start_time;
int64_t duration;
int fast;
int genpts;
int lowres;
int decoder_reorder_pts;
int autoexit;
#ifdef FFP_MERGE
int exit_on_keydown;
int exit_on_mousedown;
#endif
int loop;
int framedrop;
int64_t seek_at_start;
int subtitle;
int infinite_buffer;
enum ShowMode show_mode;
char *audio_codec_name;
char *subtitle_codec_name;
char *video_codec_name;
double rdftspeed;
#ifdef FFP_MERGE
int64_t cursor_last_shown;
int cursor_hidden;
#endif
#if CONFIG_AVFILTER
const char **vfilters_list;
int nb_vfilters;
char *afilters;
char *vfilter0;
#endif
int autorotate;
int find_stream_info;
unsigned sws_flags;
/* current context */
#ifdef FFP_MERGE
int is_full_screen;
#endif
int64_t audio_callback_time;
#ifdef FFP_MERGE
SDL_Surface *screen;
#endif
/* extra fields */
SDL_Aout *aout;
SDL_Vout *vout;
struct IJKFF_Pipeline *pipeline;
struct IJKFF_Pipenode *node_vdec;
int sar_num;
int sar_den;
char *video_codec_info;
char *audio_codec_info;
char *subtitle_codec_info;
Uint32 overlay_format;
int last_error;
int prepared;
int auto_resume;
int error;
int error_count;
int start_on_prepared;
int first_video_frame_rendered;
int first_audio_frame_rendered;
int sync_av_start;
MessageQueue msg_queue;
int64_t playable_duration_ms;
int packet_buffering;
int pictq_size;
int max_fps;
int startup_volume;
int videotoolbox;
int vtb_max_frame_width;
int vtb_async;
int vtb_wait_async;
int vtb_handle_resolution_change;
int mediacodec_all_videos;
int mediacodec_avc;
int mediacodec_hevc;
int mediacodec_mpeg2;
int mediacodec_mpeg4;
int mediacodec_handle_resolution_change;
int mediacodec_auto_rotate;
int opensles;
int soundtouch_enable;
char *iformat_name;
int no_time_adjust;
double preset_5_1_center_mix_level;
struct IjkMediaMeta *meta;
SDL_SpeedSampler vfps_sampler;
SDL_SpeedSampler vdps_sampler;
/* filters */
SDL_mutex *vf_mutex;
SDL_mutex *af_mutex;
int vf_changed;
int af_changed;
float pf_playback_rate;
int pf_playback_rate_changed;
float pf_playback_volume;
int pf_playback_volume_changed;
void *inject_opaque;
void *ijkio_inject_opaque;
FFStatistic stat;
FFDemuxCacheControl dcc;
AVApplicationContext *app_ctx;
IjkIOManagerContext *ijkio_manager_ctx;
int enable_accurate_seek;
int accurate_seek_timeout;
int mediacodec_sync;
int skip_calc_frame_rate;
int get_frame_mode;
GetImgInfo *get_img_info;
int async_init_decoder;
char *video_mime_type;
char *mediacodec_default_name;
int ijkmeta_delay_init;
int render_wait_start;
DX_RecordRelateData dx_recordRelData;
} FFPlayer;
FFPlayer中增加DX_RecordRelateData，用于保存视频录制时的相关信息。
增加录制主要实现代码
在natvie_lib的ijkmedia/ijkplayer下增加dj_record.h和dj_record.c这2个文件。
dj_record.h文件内容如下：
//
// Created by daijun on 2019-12-08.
//
#ifndef IJKPLAYER_DJ_RECORD_H
#define IJKPLAYER_DJ_RECORD_H
#include <pthread.h>
#include <android/log.h>
//#include <string>
#include <android/native_window.h>
#include "unistd.h"
//extern "C"{
//#include "ff_ffplay_def.h"
#include "libavcodec/avcodec.h"
#include "libavutil/avutil.h"
#include "libswscale/swscale.h"
#include <libavutil/imgutils.h>
#include "libswresample/swresample.h"
#include "libavutil/timestamp.h"
#include "libavutil/mathematics.h"
#include "libavutil/opt.h"
#include "libavutil/avassert.h"
//封装格式处理
#include "libavformat/avformat.h"
//像素处理
#include "libswscale/swscale.h"
//}
#define STREAM_DURATION 10.0
#define STREAM_FRAME_RATE 25 /* 25 images/s */
#define STREAM_PIX_FMT AV_PIX_FMT_YUV420P /* default pix_fmt */
#define SCALE_FLAGS SWS_BICUBIC
#define AVFMT_RAWPICTURE 0x0020 //ffmpeg源码里缺失
//自定义宏
#define DX_FRAME_TYPE_VIDEO 0
#define DX_FRAME_TYPE_AUDIO 1
#define DX_MAX_DECODE_FRAME_SIZE 3000
//录像状态
#define DX_RECORD_STATUS_OFF 0
#define DX_RECORD_STATUS_ON 1
//编码状态
#define DX_RECORD_ENCODING_OFF 0
#define DX_RECORD_ENCODING_ON 1
typedef struct OutputStream {
AVStream *st;
/* pts of the next frame that will be generated */
int64_t next_pts;
int samples_count;
AVFrame *frame;
AVFrame *tmp_frame;
float t, tincr, tincr2;
struct SwsContext *sws_ctx;
struct SwrContext *swr_ctx;
} OutputStream;
typedef struct InputSourceInfo{
int width, height;
}InputSourceInfo;
//存储解码后的音频/视频数据
typedef struct DX_FrameData{
uint8_t * data0;
uint8_t * data1;
uint8_t * data2;
int lineSize0;
int lineSize1;
int lineSize2;
//指示有几个data
int dataNum;
//帧数据类型0-视频 1-音频
int frameType;
//音频number of audio samples (per channel)
int nb_samples;
uint64_t channel_layout;
int channels;
//代表音频/视频流的格式
int format;
}DX_FrameData;
//录像相关信息
typedef struct DX_RecordRelateData{
//示例代码中编码相关
//输出上下文
OutputStream video_st;
OutputStream audio_st;
AVFormatContext *oc;
AVOutputFormat *fmt;
AVCodec *audio_codec, *video_codec;
// 录像文件名
const char *fileName;
//输入流格式
InputSourceInfo srcFormat;
// 保存的解码帧
DX_FrameData recordFramesQueue[DX_MAX_DECODE_FRAME_SIZE];
// 与recordFramesQueue相关 可读的索引值（即准备编码时取的索引）初始值0 （不考虑复用情况，可不用）
int rindex;
// 与recordFramesQueue相关 可写的索引值（即解码时写入的索引）初始值0 （即保存的解码帧个数）
int windex;
// 是否正在进行录制
int isInRecord;
// 录像线程id
pthread_t recThreadid;
// 是否正在录制数据编码中
int isOnEncoding;
}DX_RecordRelateData;
void add_stream(OutputStream *ost, AVFormatContext *oc,
AVCodec **codec,
enum AVCodecID codec_id,InputSourceInfo inputSrcInfo);
void open_video(AVFormatContext *oc, AVCodec *codec, OutputStream *ost, AVDictionary *opt_arg);
void open_audio(AVFormatContext *oc, AVCodec *codec, OutputStream *ost, AVDictionary *opt_arg);
AVFrame *dx_alloc_picture(enum AVPixelFormat pix_fmt, int width, int height);
AVFrame *alloc_audio_frame(enum AVSampleFormat sample_fmt,
uint64_t channel_layout,
int sample_rate, int nb_samples);
int write_video_frame(AVFormatContext *oc, OutputStream *ost,AVFrame * curFr);
int write_frame(AVFormatContext *fmt_ctx, const AVRational *time_base, AVStream *st, AVPacket *pkt);
void log_packet(const AVFormatContext *fmt_ctx, const AVPacket *pkt);
int write_audio_frame(AVFormatContext *oc, OutputStream *ost,AVFrame * curFr);
void close_stream(AVFormatContext *oc, OutputStream *ost);
//执行录像文件
void* doRecordFile(void *infoData);
void free_record_frames(DX_RecordRelateData* recData);
#endif //IJKPLAYER_DJ_RECORD_H
dj_record.c文件内容如下：
//
// Created by daijun on 2019-12-08.
//
#include "dj_record.h"
#define LOGD(FORMAT,...) __android_log_print(ANDROID_LOG_INFO,"dj_record",FORMAT,##__VA_ARGS__);
#define LOGE(FORMAT,...) __android_log_print(ANDROID_LOG_ERROR,"dj_record",FORMAT,##__VA_ARGS__);
void add_stream(OutputStream *ost, AVFormatContext *oc, AVCodec **codec,
enum AVCodecID codec_id,InputSourceInfo inpSrcInfo) {
AVCodecContext *c;
int i;
//find the encoder
*codec = avcodec_find_encoder(codec_id);
if (!(*codec)) {
fprintf(stderr, "Could not find encoder for '%s'\n",
avcodec_get_name(codec_id));
LOGE("未找到编码器encoder %s",avcodec_get_name(codec_id));
exit(1);
}
ost->st = avformat_new_stream(oc, *codec);
if (!ost->st) {
fprintf(stderr, "Could not allocate stream\n");
exit(1);
}
ost->st->id = oc->nb_streams-1;
c = ost->st->codec;
switch ((*codec)->type) {
case AVMEDIA_TYPE_AUDIO:
c->sample_fmt = (*codec)->sample_fmts ?
(*codec)->sample_fmts[0] : AV_SAMPLE_FMT_FLTP;
c->bit_rate = 64000;
c->sample_rate = 44100;
if ((*codec)->supported_samplerates) {
c->sample_rate = (*codec)->supported_samplerates[0];
for (i = 0; (*codec)->supported_samplerates[i]; i++) {
if ((*codec)->supported_samplerates[i] == 44100)
c->sample_rate = 44100;
}
}
c->channels = av_get_channel_layout_nb_channels(c->channel_layout);
c->channel_layout = AV_CH_LAYOUT_STEREO;
if ((*codec)->channel_layouts) {
c->channel_layout = (*codec)->channel_layouts[0];
for (i = 0; (*codec)->channel_layouts[i]; i++) {
if ((*codec)->channel_layouts[i] == AV_CH_LAYOUT_STEREO)
c->channel_layout = AV_CH_LAYOUT_STEREO;
}
}
c->channels = av_get_channel_layout_nb_channels(c->channel_layout);
ost->st->time_base = (AVRational){ 1, c->sample_rate };
break;
case AVMEDIA_TYPE_VIDEO:
c->codec_id = codec_id;
c->bit_rate = 400000;
//Resolution must be a multiple of two.
c->width = inpSrcInfo.width;
c->height = inpSrcInfo.height;
// timebase: This is the fundamental unit of time (in seconds) in terms
// of which frame timestamps are represented. For fixed-fps content,
// timebase should be 1/framerate and timestamp increments should be
// identical to 1.
ost->st->time_base = (AVRational){ 1, STREAM_FRAME_RATE };
c->time_base = ost->st->time_base;
c->gop_size = 12;
// emit one intra frame every twelve frames at most
c->pix_fmt = STREAM_PIX_FMT;
if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
// just for testing, we also add B frames
c->max_b_frames = 2;
}
if (c->codec_id == AV_CODEC_ID_MPEG1VIDEO) {
// Needed to avoid using macroblocks in which some coeffs overflow.
// This does not happen with normal video, it just happens here as
// the motion of the chroma plane does not match the luma plane.
c->mb_decision = 2;
}
break;
default:
break;
}
// Some formats want stream headers to be separate.
if (oc->oformat->flags & AVFMT_GLOBALHEADER)
c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
void open_video(AVFormatContext *oc, AVCodec *codec, OutputStream *ost,
AVDictionary *opt_arg) {
int ret;
AVCodecContext *c = ost->st->codec;
AVDictionary *opt = NULL;
av_dict_copy(&opt, opt_arg, 0);
// open the codec
ret = avcodec_open2(c, codec, &opt);
av_dict_free(&opt);
if (ret < 0) {
fprintf(stderr, "Could not open video codec: %s\n", av_err2str(ret));
LOGE("dj Could not open video codec:");
exit(1);
}
// allocate and init a re-usable frame
ost->frame = dx_alloc_picture(c->pix_fmt, c->width, c->height);
if (!ost->frame) {
fprintf(stderr, "Could not allocate video frame\n");
exit(1);
}
// If the output format is not YUV420P, then a temporary YUV420P
// picture is needed too. It is then converted to the required
// output format.
ost->tmp_frame = NULL;
if (c->pix_fmt != AV_PIX_FMT_YUV420P) {
ost->tmp_frame = dx_alloc_picture(AV_PIX_FMT_YUV420P, c->width, c->height);
if (!ost->tmp_frame) {
fprintf(stderr, "Could not allocate temporary picture\n");
exit(1);
}
}
}
void open_audio(AVFormatContext *oc, AVCodec *codec, OutputStream *ost,
AVDictionary *opt_arg) {
AVCodecContext *c;
int nb_samples;
int ret;
AVDictionary *opt = NULL;
c = ost->st->codec;
// open it
av_dict_copy(&opt, opt_arg, 0);
ret = avcodec_open2(c, codec, &opt);
av_dict_free(&opt);
if (ret < 0) {
fprintf(stderr, "Could not open audio codec: %s\n", av_err2str(ret));
LOGE("Could not open audio codec:");
exit(1);
}
// init signal generator
ost->t = 0;
ost->tincr = 2 * M_PI * 110.0 / c->sample_rate;
// increment frequency by 110 Hz per second
ost->tincr2 = 2 * M_PI * 110.0 / c->sample_rate / c->sample_rate;
if (c->codec->capabilities & AV_CODEC_CAP_VARIABLE_FRAME_SIZE)
nb_samples = 10000;
else
nb_samples = c->frame_size;
ost->frame = alloc_audio_frame(c->sample_fmt, c->channel_layout,
c->sample_rate, nb_samples);
ost->tmp_frame = alloc_audio_frame(AV_SAMPLE_FMT_FLTP, c->channel_layout,
c->sample_rate, nb_samples);
// create resampler context
ost->swr_ctx = swr_alloc();
if (!ost->swr_ctx) {
fprintf(stderr, "Could not allocate resampler context\n");
exit(1);
}
// set options
av_opt_set_int (ost->swr_ctx, "in_channel_count", c->channels, 0);
av_opt_set_int (ost->swr_ctx, "in_sample_rate", c->sample_rate, 0);
av_opt_set_sample_fmt(ost->swr_ctx, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
av_opt_set_int (ost->swr_ctx, "out_channel_count", c->channels, 0);
av_opt_set_int (ost->swr_ctx, "out_sample_rate", c->sample_rate, 0);
av_opt_set_sample_fmt(ost->swr_ctx, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
// initialize the resampling context
if ((ret = swr_init(ost->swr_ctx)) < 0) {
fprintf(stderr, "Failed to initialize the resampling context\n");
exit(1);
}
}
AVFrame* dx_alloc_picture(enum AVPixelFormat pix_fmt, int width, int height) {
AVFrame *picture;
int ret;
picture = av_frame_alloc();
if (!picture)
return NULL;
picture->format = pix_fmt;
picture->width = width;
picture->height = height;
// allocate the buffers for the frame data
ret = av_frame_get_buffer(picture, 32);
if (ret < 0) {
fprintf(stderr, "Could not allocate frame data.\n");
exit(1);
}
return picture;
}
AVFrame* alloc_audio_frame(enum AVSampleFormat sample_fmt, uint64_t channel_layout,
int sample_rate, int nb_samples) {
AVFrame *frame = av_frame_alloc();
int ret;
if (!frame) {
fprintf(stderr, "Error allocating an audio frame\n");
exit(1);
}
frame->format = sample_fmt;
frame->channel_layout = channel_layout;
frame->sample_rate = sample_rate;
frame->nb_samples = nb_samples;
if (nb_samples) {
ret = av_frame_get_buffer(frame, 0);
if (ret < 0) {
fprintf(stderr, "Error allocating an audio buffer\n");
exit(1);
}
}
return frame;
}
int write_video_frame(AVFormatContext *oc, OutputStream *ost,AVFrame * curFr) {
int ret;
AVCodecContext *c;
AVFrame *frame;
int got_packet = 0;
c = ost->st->codec;
// frame = get_video_frame(ost);
frame = curFr;
if (oc->oformat->flags & AVFMT_RAWPICTURE) {
// a hack to avoid data copy with some raw video muxers
AVPacket pkt;
av_init_packet(&pkt);
if (!frame)
return 1;
pkt.flags |= AV_PKT_FLAG_KEY;
pkt.stream_index = ost->st->index;
pkt.data = (uint8_t *)frame;
pkt.size = sizeof(AVPicture);
pkt.pts = pkt.dts = frame->pts;
av_packet_rescale_ts(&pkt, c->time_base, ost->st->time_base);
ret = av_interleaved_write_frame(oc, &pkt);
} else {
AVPacket pkt = { 0 };
av_init_packet(&pkt);
// encode the image
ret = avcodec_encode_video2(c, &pkt, frame, &got_packet);
if (ret < 0) {
fprintf(stderr, "Error encoding video frame: %s\n", av_err2str(ret));
exit(1);
}
if (got_packet) {
ret = write_frame(oc, &c->time_base, ost->st, &pkt);
} else {
ret = 0;
}
}
if (ret < 0) {
fprintf(stderr, "Error while writing video frame: %s\n", av_err2str(ret));
exit(1);
}
return (frame || got_packet) ? 0 : 1;
}
int write_frame(AVFormatContext *fmt_ctx, const AVRational *time_base, AVStream *st,
AVPacket *pkt) {
// rescale output packet timestamp values from codec to stream timebase
av_packet_rescale_ts(pkt, *time_base, st->time_base);
pkt->stream_index = st->index;
// Write the compressed frame to the media file.
log_packet(fmt_ctx, pkt);
return av_interleaved_write_frame(fmt_ctx, pkt);
}
void log_packet(const AVFormatContext *fmt_ctx, const AVPacket *pkt) {
AVRational *time_base = &fmt_ctx->streams[pkt->stream_index]->time_base;
printf("pts:%s pts_time:%s dts:%s dts_time:%s duration:%s duration_time:%s stream_index:%d\n",
av_ts2str(pkt->pts), av_ts2timestr(pkt->pts, time_base),
av_ts2str(pkt->dts), av_ts2timestr(pkt->dts, time_base),
av_ts2str(pkt->duration), av_ts2timestr(pkt->duration, time_base),
pkt->stream_index);
}
// 实际编码时需要AV_SAMPLE_FMT_FLTP的音频采样数据，而目前的音频解码后的数据已经是AV_SAMPLE_FMT_FLTP，
// 所以就不需要再进行音频格式转换。
int write_audio_frame(AVFormatContext *oc, OutputStream *ost,AVFrame * curFr) {
AVCodecContext *c;
AVPacket pkt = { 0 }; // data and size must be 0;
AVFrame *frame;
int ret;
int got_packet;
int dst_nb_samples;
av_init_packet(&pkt);
c = ost->st->codec;
// frame = get_audio_frame(ost);
//换成从外部传入数据已处理好的音频帧
frame = curFr;
// dst_nb_samples = curFr->nb_samples;
if (frame) {
if(frame->format == AV_SAMPLE_FMT_S16){ //FMT_S16进行格式转换
frame->pts = ost->next_pts;
// ost->next_pts += frame->nb_samples;
LOGE("写audio fmt-s16")
// convert samples from native format to destination codec format, using the resampler
// compute destination number of samples
dst_nb_samples = av_rescale_rnd(swr_get_delay(ost->swr_ctx, c->sample_rate) + frame->nb_samples,
c->sample_rate, c->sample_rate, AV_ROUND_UP);
av_assert0(dst_nb_samples == frame->nb_samples);
// when we pass a frame to the encoder, it may keep a reference to it
// internally;
// make sure we do not overwrite it here
ret = av_frame_make_writable(ost->frame);
if (ret < 0)
exit(1);
// convert to destination format
ret = swr_convert(ost->swr_ctx,
ost->frame->data, dst_nb_samples,
(const uint8_t **)frame->data, frame->nb_samples);
if (ret < 0) {
fprintf(stderr, "Error while converting\n");
exit(1);
}
frame = ost->frame;
ost->next_pts += dst_nb_samples;
}else{ // AV_SAMPLE_FMT_FLTP 格式
dst_nb_samples = curFr->nb_samples;
}
frame->pts = av_rescale_q(ost->samples_count, (AVRational){1, c->sample_rate}, c->time_base);
ost->samples_count += dst_nb_samples;
}
ret = avcodec_encode_audio2(c, &pkt, frame, &got_packet);
if (ret < 0) {
fprintf(stderr, "Error encoding audio frame: %s\n", av_err2str(ret));
exit(1);
}
if (got_packet) {
ret = write_frame(oc, &c->time_base, ost->st, &pkt);
if (ret < 0) {
fprintf(stderr, "Error while writing audio frame: %s\n",
av_err2str(ret));
exit(1);
}
}
return (frame || got_packet) ? 0 : 1;
}
void close_stream(AVFormatContext *oc, OutputStream *ost) {
avcodec_close(ost->st->codec);
av_frame_free(&ost->frame);
av_frame_free(&ost->tmp_frame);
sws_freeContext(ost->sws_ctx);
swr_free(&ost->swr_ctx);
}
// 方式2 直接使用解码后的yuv420p数据
void* doRecordFile(void *infoData){
// usleep(1000);
sleep(1);
DX_RecordRelateData *recordRelateDataPtr = (DX_RecordRelateData*)(infoData);
recordRelateDataPtr->isOnEncoding = DX_RECORD_ENCODING_ON;
LOGE("线程中执行C函数执行停止执行录制 %s",recordRelateDataPtr->fileName);
// free_record_frames();
LOGE("已录制Frame%d",recordRelateDataPtr->windex)
const char *fileName = recordRelateDataPtr->fileName;
//获取保存到的输入上下文
//输出上下文
AVFormatContext *oc;
AVOutputFormat *fmt; //临时变量
OutputStream* videoStPtr;
OutputStream* audioStPtr;
// AVCodec *audio_codec, *video_codec;
int ret;
int have_video = 0, have_audio = 0;
int encode_video = 0, encode_audio = 0;
AVDictionary *opt = NULL;
// av_dict_set(&opt,"profile","baseline",AV_DICT_MATCH_CASE);
av_dict_set(&opt, "preset", "veryfast", 0); // av_opt_set(pCodecCtx->priv_data,"preset","fast",0);
av_dict_set(&opt, "tune", "zerolatency", 0);
LOGE("线程中avformat_alloc_output_context2");
avformat_alloc_output_context2(&(recordRelateDataPtr->oc),NULL,NULL,fileName);
oc = recordRelateDataPtr->oc;
if (!oc){
LOGD("startRecord avformat_alloc_output_context2 fail");
avformat_alloc_output_context2(&oc, NULL, "mp4", fileName);
}
if (!oc){
LOGE("avformat_alloc_output_context2 失败");
return NULL;
}else{
LOGE("avformat_alloc_output_context2 成功");
}
fmt = recordRelateDataPtr->oc->oformat;
//Add the audio and video streams using the default format codecs
// and initialize the codecs.
LOGE("线程中add_stream");
if (fmt->video_codec != AV_CODEC_ID_NONE) {
add_stream(&(recordRelateDataPtr->video_st), recordRelateDataPtr->oc, &(recordRelateDataPtr->video_codec), fmt->video_codec,recordRelateDataPtr->srcFormat);
have_video = 1;
encode_video = 1;
}
if (fmt->audio_codec != AV_CODEC_ID_NONE) {
add_stream(&(recordRelateDataPtr->audio_st), recordRelateDataPtr->oc, &recordRelateDataPtr->audio_codec, fmt->audio_codec,recordRelateDataPtr->srcFormat);
have_audio = 1;
encode_audio = 1;
}
LOGE("线程中 open video");
videoStPtr = &recordRelateDataPtr->video_st;
audioStPtr = &recordRelateDataPtr->audio_st;
if (have_video)
open_video(recordRelateDataPtr->oc, recordRelateDataPtr->video_codec, &recordRelateDataPtr->video_st, opt);
if (have_audio)
open_audio(recordRelateDataPtr->oc, recordRelateDataPtr->audio_codec, &recordRelateDataPtr->audio_st, opt);
av_dump_format(recordRelateDataPtr->oc, 0, fileName, 1);
//open the output file, if needed
LOGE("线程中 avio_open");
if (!(fmt->flags & AVFMT_NOFILE)) {
ret = avio_open(&oc->pb, fileName, AVIO_FLAG_WRITE);
if (ret < 0) {
LOGE("Could not open %s",fileName);
LOGE("dj Could not open %s",fileName);
return NULL;
}
}
//Write the stream header, if any.
LOGE("线程中 avformat_write_header");
ret = avformat_write_header(oc, &opt);
if (ret < 0) {
LOGE("Error occurred when opening output file %s",av_err2str(ret));
return NULL;
}
InputSourceInfo inSrcInfo = recordRelateDataPtr->srcFormat;
//v3 根据所有解码后的帧顺序(包括视频帧和音频帧来编码)
int frNum = recordRelateDataPtr->windex;
LOGE("总共需编码 %d 帧",frNum);
struct SwsContext *swsContext = sws_getContext(
inSrcInfo.width //原图片的宽
,inSrcInfo.height //源图高
,AV_PIX_FMT_YUV420P //源图片format
,inSrcInfo.width //目标图的宽
,inSrcInfo.height //目标图的高
,AV_PIX_FMT_YUV420P,SWS_FAST_BILINEAR
, NULL, NULL, NULL
);
// 音频AV_SAMPLE_FMT_S16 转 AV_SAMPLE_FMT_FLTP
// struct SwrContext *swrAudio = swr_alloc();
// av_opt_set_int (swrAudio, "in_channel_count", AV_CH_LAYOUT_STEREO, 0);
// av_opt_set_int (swrAudio, "in_sample_rate", 16000, 0);
// av_opt_set_sample_fmt(swrAudio, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
// av_opt_set_int (swrAudio, "out_channel_count", AV_CH_LAYOUT_STEREO, 0);
// av_opt_set_int (swrAudio, "out_sample_rate", 16000, 0);
// av_opt_set_sample_fmt(swrAudio, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
// swr_init(swrAudio);
//// swrAudio = swr_alloc();
LOGE("线程中 开始音视频编码");
// for(int i=0;i<recordRelateDataPtr->recordDataFrames.size();i++){ // method 1
for(int i=0;i<recordRelateDataPtr->windex;i++){ //method 2
// DX_FrameData frData = recordRelateDataPtr->recordDataFrames[i]; // method 1
DX_FrameData frData = recordRelateDataPtr->recordFramesQueue[i]; //method 2
// LOGE("线程中 拿到一解码帧数据");
// 视频帧
if(frData.frameType == DX_FRAME_TYPE_VIDEO){
//对输出图像进行色彩，分辨率缩放，滤波处理
uint8_t *srcSlice[3];
srcSlice[0] = frData.data0;
srcSlice[1] = frData.data1;
srcSlice[2] = frData.data2;
int srcLineSize[3];
srcLineSize[0] = frData.lineSize0;
srcLineSize[1] = frData.lineSize1;
srcLineSize[2] = frData.lineSize2;
//yuv数据每个分量一个字节，一行大小等于宽度
int dstLineSize[3];
dstLineSize[0] = inSrcInfo.width;
dstLineSize[1] = inSrcInfo.width/2;
dstLineSize[2] = inSrcInfo.width/2;
sws_scale(swsContext, (const uint8_t *const *) srcSlice, srcLineSize, 0,
inSrcInfo.height, recordRelateDataPtr->video_st.frame->data, recordRelateDataPtr->video_st.frame->linesize);
recordRelateDataPtr->video_st.frame->pts = recordRelateDataPtr->video_st.next_pts++;
// fill_yuv_image(recordRelateDataPtr->video_st.frame,i,inSrcInfo.width,inSrcInfo.height);
LOGE("线程中 开始处理一帧视频数据");
write_video_frame(oc,&recordRelateDataPtr->video_st,recordRelateDataPtr->video_st.frame);
LOGE("线程中 完成一帧数据编码写入");
}else { //音频帧
//avcodec_decode_audio4 解码出来的音频数据是 AV_SAMPLE_FMT_FLTP,所以数据在data0 data1中
// LOGE("开始编码音频帧 nb_samples %d channels %d channel_layout %d",frData.nb_samples,frData.channels,frData.channel_layout);
if(frData.format == AV_SAMPLE_FMT_FLTP){
AVFrame* tmpFr = audioStPtr->tmp_frame;
tmpFr->data[0] = (uint8_t *)av_malloc(frData.lineSize0);
tmpFr->data[1] = (uint8_t *)av_malloc(frData.lineSize1);
memcpy(tmpFr->data[0],frData.data0,frData.lineSize0);
memcpy(tmpFr->data[1],frData.data1,frData.lineSize1);
tmpFr->nb_samples = frData.nb_samples;
tmpFr->channels = frData.channels;
tmpFr->channel_layout = frData.channel_layout;
tmpFr->pts = audioStPtr->next_pts;
audioStPtr->next_pts += tmpFr->nb_samples;
// LOGE("完成单帧音频编码数据拷贝");
// LOGE("线程中 开始写视频帧");
tmpFr->format = frData.format;
write_audio_frame(oc,audioStPtr,tmpFr);
}else if(frData.format == AV_SAMPLE_FMT_S16){ //需要音频格式转换
LOGE("线程中 开始处理一帧音频数据");
AVFrame* tmpFr = audioStPtr->tmp_frame;
// tmpFr->pts = audioStPtr->next_pts;
tmpFr->nb_samples = frData.nb_samples;
tmpFr->channels = frData.channels;
tmpFr->channel_layout = frData.channel_layout;
tmpFr->linesize[0] = frData.lineSize0;
// audioStPtr->next_pts += tmpFr->nb_samples;
// 直接拷贝数据
tmpFr->data[0] = (uint8_t *)av_malloc(frData.lineSize0);
memcpy(tmpFr->data[0],frData.data0,frData.lineSize0);
tmpFr->format = frData.format;
write_audio_frame(oc,audioStPtr,tmpFr);
}
}
}
LOGE("线程中 音视频编码完毕");
free_record_frames(recordRelateDataPtr);
//Write the trailer, if any. The trailer must be written before you
//close the CodecContexts open when you wrote the header; otherwise
//av_write_trailer() may try to use memory that was freed on
//av_codec_close().
av_write_trailer(oc);
//Close each codec.
if (have_video)
close_stream(oc, videoStPtr);
if (have_audio)
close_stream(oc, audioStPtr);
if (!(fmt->flags & AVFMT_NOFILE))
//Close the output file.
avio_closep(&oc->pb);
//free the stream
avformat_free_context(oc);
recordRelateDataPtr->isOnEncoding = DX_RECORD_ENCODING_OFF;
return NULL;
}
void free_record_frames(DX_RecordRelateData* recData) {
LOGE("C线程中执行录像内存释放");
LOGE("recordDaraFrameQueue size=%d",recData->windex);
//使用数组保存时的内存释放
for(int i= 0;i<recData->windex;i++){
DX_FrameData dataFrame = recData->recordFramesQueue[i];
uint8_t *data0 = dataFrame.data0;
uint8_t *data1 = dataFrame.data1;
uint8_t *data2 = dataFrame.data2;
if(dataFrame.dataNum == 1){
if(data0 != NULL){
free(data0);
data0 = NULL;
}
}else if(dataFrame.dataNum == 2){
if (data1 != NULL){
free(data1);
data1 = NULL;
}
}else if(dataFrame.dataNum == 3){
if(data2 != NULL){
free(data2);
data2 = NULL;
}
}
}
recData->windex = 0;
}
整体实现就是将解码后的视频yuv、音频采样数据保存起来，然后再进行编码，封装成视频文件。
android工程代码：
https://github.com/godtrace12/ijkplayer
contrib目录文件：
https://github.com/godtrace12/contrib
网友评论