1 前言
以camera录制过程为例，分析视频音频编解码创建流程。
2 关键log
#设置参数
07-28 03:44:05.117 14402 14402 V MediaRecorder: setAudioSource(1)
07-28 03:44:05.121 14402 14402 V MediaRecorder: setVideoSource(2)
07-28 03:44:05.122 14402 14402 V MediaRecorder: setOutputFormat(2)
07-28 03:44:05.123 14402 14402 V MediaRecorder: setVideoFrameRate(30)
07-28 03:44:05.125 14402 14402 V MediaRecorder: setParameters(video-param-encoding-bitrate=20000000)
07-28 03:44:05.126 14402 14402 V MediaRecorder: setVideoSize(1920, 1080)
07-28 03:44:05.126 14402 14402 V MediaRecorder: setVideoEncoder(3)
07-28 03:44:05.126 14402 14402 V MediaRecorder: setParameters(audio-param-encoding-bitrate=156000)
07-28 03:44:05.126 14402 14402 V MediaRecorder: setParameters(audio-param-number-of-channels=2)
07-28 03:44:05.126 14402 14402 V MediaRecorder: setParameters(audio-param-sampling-rate=48000)
07-28 03:44:05.126 14402 14402 V MediaRecorder: setAudioEncoder(3)
07-28 03:44:05.127 14402 14402 V MediaRecorder: setParameters(max-duration=600000)
07-28 03:44:05.128 14402 14402 V MediaRecorder: setParameters(max-filesize=11065696640)
07-28 03:44:05.128 14402 14402 V MediaRecorder: setParameters(video-param-rotation-angle-degrees=0)
07-28 03:44:05.129 14402 14402 V MediaRecorder: setOutputFile(79)# 获取mediaprofile信息
07-28 03:44:05.257 945 3290 V StagefrightRecorder: clipAudioBitRate: encoder 3
07-28 03:44:05.257 945 3290 V MediaProfiles: getAudioEncoderParamByName: enc.aud.bps.min for codec 3
07-28 03:44:05.257 945 3290 V MediaProfiles: getAudioEncoderParamByName: enc.aud.bps.max for codec 3
07-28 03:44:05.257 945 3290 V StagefrightRecorder: clipAudioSampleRate: encoder 3
07-28 03:44:05.257 945 3290 V MediaProfiles: getAudioEncoderParamByName: enc.aud.hz.min for codec 3
07-28 03:44:05.257 945 3290 V MediaProfiles: getAudioEncoderParamByName: enc.aud.hz.max for codec 3# 准备
08-10 03:16:48.044 3470 3470 V MediaRecorder: prepare
08-10 03:16:48.044 950 1864 V StagefrightRecorder: prepare
08-10 03:16:48.044 950 1864 V StagefrightRecorder: prepare
08-10 03:16:48.044 950 1864 V MPEG4Writer: initInternal
08-10 03:16:48.045 722 1379 V AudioFlinger: onIdleMixer
08-10 03:16:48.045 950 1864 D MediaCodecSource: youkai in MediaCodecSource::initEncoder
08-10 03:16:48.045 950 1864 V MediaCodecList: youkai in MediaCodecList::findMatchingCodecs
08-10 03:16:48.045 950 1864 V MediaCodecList: componentName 'OMX.qcom.video.encoder.avc'
08-10 03:16:48.045 950 1864 V MediaCodecList: matching 'OMX.qcom.video.encoder.avc'
08-10 03:16:48.046 950 1864 V MediaCodecList: componentName 'c2.android.avc.encoder'
08-10 03:16:48.046 950 1864 V MediaCodecList: matching 'c2.android.avc.encoder'
08-10 03:16:48.046 950 1864 V MediaCodecList: componentName 'OMX.google.h264.encoder'
08-10 03:16:48.046 950 1864 V MediaCodecList: matching 'OMX.google.h264.encoder'
08-10 03:16:48.046 950 1864 V MediaCodecSource: matchingCodecs.size() is '3'
08-10 03:16:48.046 950 1864 V MediaCodecSource: youkai in for-----
08-10 03:16:48.046 950 1864 D MediaCodec: youkai in MediaCodec::MediaCodec
08-10 03:16:48.046 950 1864 D MediaCodec: youkai in MediaCodec::CreateByComponentName
08-10 03:16:48.046 950 1864 D MediaCodec: youkai in MediaCodec::init
08-10 03:16:48.047 950 1864 D MediaCodec: youkai in MediaCodec::GetCodecBase
08-10 03:16:48.047 950 1864 D ACodec : youkai in Acodec::Acodec
08-10 03:16:48.047 950 1864 V ACodec : Now uninitialized
08-10 03:16:48.048 950 3676 V ACodec : onAllocateComponent
08-10 03:16:48.051 950 3676 I OMXClient: IOmx service obtained
08-10 03:16:48.051 975 2052 I OMXMaster: makeComponentInstance(OMX.qcom.video.encoder.avc) in android.hardwar process08-10 03:16:48.089 950 3676 V ACodec : [OMX.qcom.video.encoder.avc] Now Loaded
08-10 03:16:48.091 950 1864 V MediaCodecSource: output format is AMessage(what = 0x00000000) = {
08-10 03:16:48.091 950 1864 V MediaCodecSource: string mime = "video/avc"
08-10 03:16:48.091 950 1864 V MediaCodecSource: int32_t width = 1920
08-10 03:16:48.091 950 1864 V MediaCodecSource: int32_t height = 1080
08-10 03:16:48.091 950 1864 V MediaCodecSource: int32_t stride = 1920
08-10 03:16:48.091 950 1864 V MediaCodecSource: int32_t slice-height = 1080
08-10 03:16:48.091 950 1864 V MediaCodecSource: int32_t color-format = 2130708361
08-10 03:16:48.091 950 1864 V MediaCodecSource: int32_t bitrate = 20000000
08-10 03:16:48.091 950 1864 V MediaCodecSource: int32_t frame-rate = 30
08-10 03:16:48.091 950 1864 V MediaCodecSource: int32_t i-frame-interval = 1
08-10 03:16:48.091 950 1864 V MediaCodecSource: int32_t priority = 0
08-10 03:16:48.091 950 1864 V MediaCodecSource: int32_t feature-nal-length-bitstream = 1
08-10 03:16:48.091 950 1864 V MediaCodecSource: int32_t nal-length-in-bytes = 4
08-10 03:16:48.091 950 1864 V MediaCodecSource: int32_t isNativeRecorder = 1
08-10 03:16:48.091 950 1864 V MediaCodecSource: int32_t create-input-buffers-suspended = 1
08-10 03:16:48.091 950 1864 V MediaCodecSource: }'
08-10 03:16:48.091 950 3675 I MediaCodec: MediaCodec will operate in async mode
08-10 03:16:48.093 950 3675 V MediaCodec: kWhatConfigure: Old mCrypto: 0x0 (0)
08-10 03:16:48.093 950 3675 V MediaCodec: kWhatConfigure: New mCrypto: 0x0 (0)
08-10 03:16:48.093 950 3675 V MediaCodec: Found 0 pieces of codec specific data.
08-10 03:16:48.093 950 3676 V ACodec : onConfigureComponent08-10 03:16:48.144 950 3675 V MediaCodec: [OMX.qcom.video.encoder.avc] configured as input format: AMessage(what = 0x00000000) = {
08-10 03:16:48.144 950 3675 V MediaCodec: string mime = "video/raw"
08-10 03:16:48.144 950 3675 V MediaCodec: int32_t stride = 1920
08-10 03:16:48.144 950 3675 V MediaCodec: int32_t slice-height = 1080
08-10 03:16:48.144 950 3675 V MediaCodec: int32_t color-format = 2130708361
08-10 03:16:48.144 950 3675 V MediaCodec: int32_t color-range = 0
08-10 03:16:48.144 950 3675 V MediaCodec: int32_t color-standard = 0
08-10 03:16:48.144 950 3675 V MediaCodec: int32_t color-transfer = 0
08-10 03:16:48.144 950 3675 V MediaCodec: int32_t width = 1920
08-10 03:16:48.144 950 3675 V MediaCodec: int32_t height = 1080
08-10 03:16:48.144 950 3675 V MediaCodec: }, output format: AMessage(what = 0x00000000) = {
08-10 03:16:48.144 950 3675 V MediaCodec: int32_t bitrate = 20000000
08-10 03:16:48.144 950 3675 V MediaCodec: int32_t max-bitrate = 20000000
08-10 03:16:48.144 950 3675 V MediaCodec: int32_t frame-rate = 30
08-10 03:16:48.144 950 3675 V MediaCodec: string mime = "video/avc"
08-10 03:16:48.144 950 3675 V MediaCodec: int32_t width = 1920
08-10 03:16:48.144 950 3675 V MediaCodec: int32_t height = 1080
08-10 03:16:48.144 950 3675 V MediaCodec: }
08-10 03:16:48.145 950 1864 V MediaCodecSource: youkai out for-----3 MediaCodecSource::initEncoder
status_t MediaCodecSource::initEncoder() {
//1. 获取outputMIME，用于检索encode名字AString outputMIME;CHECK(mOutputFormat->findString("mime", &outputMIME));//2. 根据outputMIME检索匹配的encode名字，传入matchingCodecsMediaCodecList::findMatchingCodecs(outputMIME.c_str(), true /* encoder */,((mFlags & FLAG_PREFER_SOFTWARE_CODEC) ? MediaCodecList::kPreferSoftwareCodecs : 0),&matchingCodecs);//3. 创建mediacodecmEncoder = MediaCodec::CreateByComponentName(mCodecLooper, matchingCodecs[ix]);//4. 打印输出类型logALOGV("output format is '%s'", mOutputFormat->debugString(0).c_str());//5. 配置codecerr = mEncoder->configure(mOutputFormat,NULL /* nativeWindow */,NULL /* crypto */,MediaCodec::CONFIGURE_FLAG_ENCODE);
}
3.1 获取mime
AString outputMIME;CHECK(mOutputFormat->findString("mime", &outputMIME));mIsVideo = outputMIME.startsWithIgnoreCase("video/");
mOutputFormat -> outputMIME
status_t StagefrightRecorder::setupVideoEncoder(const sp<MediaSource> &cameraSource,sp<MediaCodecSource> *source) {
source->clear();ALOGE("youkai in StagefrightRecorder::setupVideoEncoder");sp<AMessage> format = new AMessage();switch (mVideoEncoder) {
case VIDEO_ENCODER_H263:format->setString("mime", MEDIA_MIMETYPE_VIDEO_H263);break;case VIDEO_ENCODER_MPEG_4_SP:format->setString("mime", MEDIA_MIMETYPE_VIDEO_MPEG4);break;case VIDEO_ENCODER_H264:format->setString("mime", MEDIA_MIMETYPE_VIDEO_AVC);break;case VIDEO_ENCODER_VP8:format->setString("mime", MEDIA_MIMETYPE_VIDEO_VP8);break;case VIDEO_ENCODER_HEVC:format->setString("mime", MEDIA_MIMETYPE_VIDEO_HEVC);break;default:CHECK(!"Should not be here, unsupported video encoding.");break;}
}//mediaDefs.cpp
const char *MEDIA_MIMETYPE_VIDEO_VP8 = "video/x-vnd.on2.vp8";
const char *MEDIA_MIMETYPE_VIDEO_VP9 = "video/x-vnd.on2.vp9";
const char *MEDIA_MIMETYPE_VIDEO_AV1 = "video/av01";
const char *MEDIA_MIMETYPE_VIDEO_AVC = "video/avc";
const char *MEDIA_MIMETYPE_VIDEO_HEVC = "video/hevc";
const char *MEDIA_MIMETYPE_VIDEO_MPEG4 = "video/mp4v-es";
const char *MEDIA_MIMETYPE_VIDEO_H263 = "video/3gpp";
const char *MEDIA_MIMETYPE_VIDEO_MPEG2 = "video/mpeg2";
const char *MEDIA_MIMETYPE_VIDEO_RAW = "video/raw";
//java代码
private int mVideoEncoder;
// Default 0. If it is larger than 0, the camcorder is in time lapse mode.mMediaRecorder.setVideoEncoder(mVideoEncoder);
这个值在应用界面上可以选择，当前支持mpeg4/h264/h265.
status_t StagefrightRecorder::setVideoEncoder(video_encoder ve) {
ALOGV("setVideoEncoder: %d", ve);if (ve < VIDEO_ENCODER_DEFAULT ||ve >= VIDEO_ENCODER_LIST_END) {
ALOGE("Invalid video encoder: %d", ve);return BAD_VALUE;}mVideoEncoder = ve;return OK;
}enum video_encoder {
VIDEO_ENCODER_DEFAULT = 0,VIDEO_ENCODER_H263 = 1,VIDEO_ENCODER_H264 = 2,VIDEO_ENCODER_MPEG_4_SP = 3,VIDEO_ENCODER_VP8 = 4,VIDEO_ENCODER_HEVC = 5,VIDEO_ENCODER_LIST_END // must be the last - used to validate the video encoder type
};StagefrightRecorder::setVideoEncoder进行判断当前设置的视频编码器选择。
小结：应用选择编码器类型VIDEO_ENCODER_MPEG_4_SP，然后在StagefrightRecorder::setVideoEncoder中进行判断，确定mime为video/mp4v-es。
3.2 检索匹配的encode名字
//static
void MediaCodecList::findMatchingCodecs(const char *mime, bool encoder, uint32_t flags,Vector<AString> *matches) {
matches->clear();const sp<IMediaCodecList> list = getInstance();size_t index = 0;for (;;) {
ssize_t matchIndex =list->findCodecByType(mime, encoder, index);if (matchIndex < 0) {
break;}index = matchIndex + 1;const sp<MediaCodecInfo> info = list->getCodecInfo(matchIndex);CHECK(info != nullptr);AString componentName = info->getCodecName();ALOGV("componentName '%s'", componentName.c_str());if ((flags & kHardwareCodecsOnly) && isSoftwareCodec(componentName)) {
ALOGV("skipping SW codec '%s'", componentName.c_str());} else {
matches->push(componentName);ALOGV("matching '%s'", componentName.c_str());}}if (flags & kPreferSoftwareCodecs ||property_get_bool("debug.stagefright.swcodec", false)) {
ALOGV("youkai in compareSoftwareCodecsFirst"); matches->sort(compareSoftwareCodecsFirst);}
}
这里先获取mediacodeclist（这是从mediacodec.xml文件中提取的信息）
将传入的mime作为依据，找到匹配的encode
<MediaCodec name="OMX.qcom.video.encoder.mpeg4" type="video/mp4v-es" ><Quirk name="requires-allocate-on-input-ports" /><Quirk name="requires-allocate-on-output-ports" /><Quirk name="requires-loaded-to-idle-after-allocation" /><Limit name="size" min="96x64" max="1920x1088" /><Limit name="alignment" value="2x2" /><Limit name="block-size" value="16x16" /><Limit name="blocks-per-second" min="1" max="244800" /><Limit name="bitrate" range="1-40000000" /><Limit name="concurrent-instances" max="16" /><Limit name="performance-point-1920x1088" value="30" /></MediaCodec>
小结：根据mime"video/mp4v-es"，在mediacodeclist中找到匹配的encode名字，存入matches中。
3.3 创建mediacodec
mEncoder = MediaCodec::CreateByComponentName(mCodecLooper, matchingCodecs[ix]);sp<MediaCodec> MediaCodec::CreateByComponentName(const sp<ALooper> &looper, const AString &name, status_t *err, pid_t pid, uid_t uid) {
sp<MediaCodec> codec = new MediaCodec(looper, pid, uid);ALOGD("youkai in MediaCodec::CreateByComponentName");const status_t ret = codec->init(name);if (err != NULL) {
*err = ret;}return ret == OK ? codec : NULL; // NULL deallocates codec.
}
MediaCodec::CreateByComponentName
-> new MediaCodec
-> MediaCodec::init
根据传入的encode名字matchingCodecs[ix]创建mediacodec。
注意在init的时候会创建Acodec
3.4 配置encode
status_t MediaCodec::configure(const sp<AMessage> &format,const sp<Surface> &surface,const sp<ICrypto> &crypto,const sp<IDescrambler> &descrambler,uint32_t flags) {
sp<AMessage> msg = new AMessage(kWhatConfigure, this);if (mIsVideo) {
format->findInt32("width", &mVideoWidth);format->findInt32("height", &mVideoHeight);if (!format->findInt32("rotation-degrees", &mRotationDegrees)) {
mRotationDegrees = 0;}if (mAnalyticsItem != NULL) {
mAnalyticsItem->setInt32(kCodecWidth, mVideoWidth);mAnalyticsItem->setInt32(kCodecHeight, mVideoHeight);mAnalyticsItem->setInt32(kCodecRotation, mRotationDegrees);int32_t maxWidth = 0;if (format->findInt32("max-width", &maxWidth)) {
mAnalyticsItem->setInt32(kCodecMaxWidth, maxWidth);}int32_t maxHeight = 0;if (format->findInt32("max-height", &maxHeight)) {
mAnalyticsItem->setInt32(kCodecMaxHeight, maxHeight);}}// Prevent possible integer overflow in downstream code.if (mVideoWidth < 0 || mVideoHeight < 0 ||(uint64_t)mVideoWidth * mVideoHeight > (uint64_t)INT32_MAX / 4) {
ALOGE("Invalid size(s), width=%d, height=%d", mVideoWidth, mVideoHeight);return BAD_VALUE;}}//重点for (int i = 0; i <= kMaxRetry; ++i) {
if (i > 0) {
// Don't try to reclaim resource for the first time.if (!mResourceManagerService->reclaimResource(resources)) {
break;}}sp<AMessage> response;err = PostAndAwaitResponse(msg, &response);if (err != OK && err != INVALID_OPERATION) {
// MediaCodec now set state to UNINITIALIZED upon any fatal error.// To maintain backward-compatibility, do a reset() to put codec// back into INITIALIZED state.// But don't reset if the err is INVALID_OPERATION, which means// the configure failure is due to wrong state.ALOGE("configure failed with err 0x%08x, resetting...", err);reset();}if (!isResourceError(err)) {
break;}}return err;
}
这里主要就是创建消息，然后把encode的相关参数存入消息中，这里重点在于PostAndAwaitResponse。
3.5 疑问
为什么当前匹配到三种，但根据log只执行了一次。
推测在于配置mediacodec的返回值，目前的三个名字，依次创建，如果第一个encode就符合，那么返回值为ok，直接退出。
需要注意的是，在configure函数中会获取profile，对上层设置的参数进行匹配，后续尝试强制修改不符合看看效果。
err = mEncoder->configure(mOutputFormat,NULL /* nativeWindow */,NULL /* crypto */,MediaCodec::CONFIGURE_FLAG_ENCODE);ALOGV("youkai out for-----");if (err == OK) {
break;}
4 总结
本文简单介绍从app设置encode类型，从而获取mime，再由mime确定匹配的encode名字，然后创建对应的mediacodec。
5 补充——强制设置encode不符合
待办