Android 13 - Media框架（30）- MediaCodec（五）

前一节我们了解了input buffer写入的流程，知道了起播写前几笔数据时会先获取graphic buffer，这一节我们就一起来了解下dequeueBufferFromNativeWindow是如何工作的。

1、dequeueBufferFromNativeWindow

ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() {
	// 判断是否有 native window
    ANativeWindowBuffer *buf;
    CHECK(mNativeWindow.get() != NULL);
	// tunnel mode无需获取graphic buffer
    if (mTunneled) {
        ALOGW("dequeueBufferFromNativeWindow() should not be called in tunnel"
              " video playback mode mode!");
        return NULL;
    }

    if (mFatalError) {
        ALOGW("not dequeuing from native window due to fatal error");
        return NULL;
    }

    int fenceFd = -1;
    do {
    	// 从 native window 获取 buffer
        status_t err = mNativeWindow->dequeueBuffer(mNativeWindow.get(), &buf, &fenceFd);
        if (err != 0) {
            ALOGE("dequeueBuffer failed: %s(%d).", asString(err), err);
            return NULL;
        }

        bool stale = false;
        for (size_t i = mBuffers[kPortIndexOutput].size(); i > 0;) {
            i--;
            // 获取一个 BufferInfo，index从大到小
            BufferInfo *info = &mBuffers[kPortIndexOutput].editItemAt(i);
			// 如果bufferinfo中的mGraphicBuffer不为null，且mGraphicBuffer的handle等于获取的buffer的handle
            if (info->mGraphicBuffer != NULL &&
                    info->mGraphicBuffer->handle == buf->handle) {
                // Since consumers can attach buffers to BufferQueues, it is possible
                // that a known yet stale buffer can return from a surface that we
                // once used.  We can simply ignore this as we have already dequeued
                // this buffer properly.  NOTE: this does not eliminate all cases,
                // e.g. it is possible that we have queued the valid buffer to the
                // NW, and a stale copy of the same buffer gets dequeued - which will
                // be treated as the valid buffer by ACodec.
                // 如果buffer的状态不是属于native window，则说明这个buffer是过时的buffer
                if (info->mStatus != BufferInfo::OWNED_BY_NATIVE_WINDOW) {
                    ALOGI("dequeued stale buffer %p. discarding", buf);
                    stale = true;
                    break;
                }

                ALOGV("dequeued buffer #%u with age %u, graphicBuffer %p",
                        (unsigned)(info - &mBuffers[kPortIndexOutput][0]),
                        mDequeueCounter - info->mDequeuedAt,
                        info->mGraphicBuffer->handle);
				// 否则直接将bufferinfo返回，同时给bufferinfo设置write fence
                info->mStatus = BufferInfo::OWNED_BY_US;
                info->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow");
                updateRenderInfoForDequeuedBuffer(buf, fenceFd, info);
                return info;
            }
        }

        // It is also possible to receive a previously unregistered buffer
        // in non-meta mode. These should be treated as stale buffers. The
        // same is possible in meta mode, in which case, it will be treated
        // as a normal buffer, which is not desirable.
        // TODO: fix this.
        if (!stale && !storingMetadataInDecodedBuffers()) {
            ALOGI("dequeued unrecognized (stale) buffer %p. discarding", buf);
            stale = true;
        }
        // 如果是过时的buffer则重新从native window获取graphic buffer
        if (stale) {
            // TODO: detach stale buffer, but there is no API yet to do it.
            buf = NULL;
        }
    } while (buf == NULL);

    // get oldest undequeued buffer
    // 计算没有使用时间最长的buffer
    BufferInfo *oldest = NULL;
    for (size_t i = mBuffers[kPortIndexOutput].size(); i > 0;) {
        i--;
        BufferInfo *info =
            &mBuffers[kPortIndexOutput].editItemAt(i);
        if (info->mStatus == BufferInfo::OWNED_BY_NATIVE_WINDOW &&
            (oldest == NULL ||
             // avoid potential issues from counter rolling over
             mDequeueCounter - info->mDequeuedAt >
                    mDequeueCounter - oldest->mDequeuedAt)) {
            oldest = info;
        }
    }

    // it is impossible dequeue a buffer when there are no buffers with ANW
    CHECK(oldest != NULL);
    // it is impossible to dequeue an unknown buffer in non-meta mode, as the
    // while loop above does not complete
    CHECK(storingMetadataInDecodedBuffers());
	// 将获取到的graphic buffer绑定到buffer info上，并且标注上是新的graphic buffer
    // discard buffer in LRU info and replace with new buffer
    oldest->mGraphicBuffer = GraphicBuffer::from(buf);
    oldest->mNewGraphicBuffer = true;
    oldest->mStatus = BufferInfo::OWNED_BY_US;
    // 设置 write fence
    oldest->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow for oldest");
    mRenderTracker.untrackFrame(oldest->mRenderInfo);
    oldest->mRenderInfo = NULL;

    ALOGV("replaced oldest buffer #%u with age %u, graphicBuffer %p",
            (unsigned)(oldest - &mBuffers[kPortIndexOutput][0]),
            mDequeueCounter - oldest->mDequeuedAt,
            oldest->mGraphicBuffer->handle);

    updateRenderInfoForDequeuedBuffer(buf, fenceFd, oldest);
    return oldest;
}

dequeueBufferFromNativeWindow 的代码比较长，但是如果了解buffer机制很容易就知道这段代码是什么意思了，接下来就来谈谈我的理解。

在开始解码流程之初，我们都知道output buffer没有指向实际的graphic buffer，而正常工作的状态下，一个 BufferInfo 会对应有一个 graphic buffer，但是这里的绑定关系可能会变化的，因为从native window 中获取的 buffer 可能是新的 buffer，这时候就要选择绑定到某一个 BufferInfo 上了。了解了这个内容，我们再来看代码：

• 调用mNativeWindow->dequeueBuffer获取graphic buffer；
• 遍历所有的 BufferInfo，查找是否有handle相同的BufferInfo，如果有就说明这个graphic buffer已经绑定过 BufferInfo 了，接下来就要检查 graphic buffer的状态，如果BufferInfo存储的状态不是OWNED_BY_NATIVE_WINDOW，就说明这块buffer已经被使用了，还没有还给native window，这块buffer是不能够再被使用的，因此需要重新dequeueBuffer；
• 如果dequeue出来的graphic buffer没有对应的 BufferInfo，说明这是一块新的 graphic buffer，需要绑定到一个 BufferInfo，绑定到哪一个上面呢？
  • ACodec 给出的方法是绑定到一个最久没有被使用的 BufferInfo 上；如何判断最久没有被使用呢？遍历所有的BufferInfo，判断归属于OWNED_BY_NATIVE_WINDOW，并且mDequeuedAt最小的BufferInfo；ACodec 有一个成员 mDequeueCounter ，decoder每次输出一帧，这个计数值会加一；BufferInfo有一个成员mDequeuedAt，这个值用于记录当前BufferInfo在第几帧被使用（填上输出）；有了这两个值，就可以知道当前哪个 BufferInfo 最久没有被使用了。
  • 最新绑定的 BufferInfo 还有一个成员 mNewGraphicBuffer 会被置为 true，说明这是一个新的 graphic buffer，需要注册给 OMX 组件。
• dequeueBuffer 获取 graphic buffer的同时会拿到一个 fence，fence翻译为篱笆，其实就是保护的意思；为什么要这个东西呢？dequeueBuffer 有一种特殊情况，graphic buffer已经被填充返回给 native window，但是buffer中的内容还没有被消费，这时候就重新被获取，并让decoder去填充；但是这很明显是有问题的，decoder需要等待 buffer 中的内容被消费完才能填充这个 buffer，但是graphic buffer的填充和使用是在两个不同的进程当中的，要如何知道buffer被消费完成呢？或者知道buffer被填充完成呢？这里就用fence来实现，填充完成时用fence加锁，等到消费完成解锁，才能继续填充，大致就是这个意思。
• FrameRenderTracker我们后面再研究是干什么用的。

到这 dequeueBufferFromNativeWindow 就分析完成了，拿到graphic buffer之后，就要把它送给decoder使用了。

2、fillBuffer

status_t ACodec::fillBuffer(BufferInfo *info) {
    status_t err;
    // Even in dynamic ANW buffer mode, if the graphic buffer is not changing,
    // send sPreset instead of the same graphic buffer, so that OMX server
    // side doesn't update the meta. In theory it should make no difference,
    // however when the same buffer is parcelled again, a new handle could be
    // created on server side, and some decoder doesn't recognize the handle
    // even if it's the same buffer.
    if (!storingMetadataInDecodedBuffers() || !info->mNewGraphicBuffer) {
        err = mOMXNode->fillBuffer(
            info->mBufferID, OMXBuffer::sPreset, info->mFenceFd);
    } else {
        err = mOMXNode->fillBuffer(
            info->mBufferID, info->mGraphicBuffer, info->mFenceFd);
    }

    info->mNewGraphicBuffer = false;
    info->mFenceFd = -1;
    if (err == OK) {
        info->mStatus = BufferInfo::OWNED_BY_COMPONENT;
    }
    return err;
}

fillBuffer 的代码比较简单，就是调用 OMXNode 的 fillBuffer 方法，但是分为两种情况：

• 当 BufferInfo 没有发生变化时，调用 fillBuffer 时传入参数为 OMXBuffer::sPreset；什么叫没有发生变化，当output buffer 使用普通buffer，buffer就不会发生变化；当graphic buffer没有发生变化，BufferInfo也视作为没有发生变化；
• 当 BufferInfo 发生变化时，调用 fillBuffer 传入参数为info->mGraphicBuffer；当BufferInfo中绑定了新的graphic buffer时，需要把graphic buffer重新绑定给decoder，因此需要作为参数传递给OMX。

3、OMXNodeInstance::fillBuffer

status_t OMXNodeInstance::fillBuffer(
        IOMX::buffer_id buffer, const OMXBuffer &omxBuffer, int fenceFd) {
    Mutex::Autolock autoLock(mLock);
    if (mHandle == NULL) {
        return DEAD_OBJECT;
    }
	// 找到id对应的 Buffer header
    OMX_BUFFERHEADERTYPE *header = findBufferHeader(buffer, kPortIndexOutput);
    if (header == NULL) {
        ALOGE("b/25884056");
        return BAD_VALUE;
    }
	// 判断buffer type是否为kBufferTypeANWBuffer
    if (omxBuffer.mBufferType == OMXBuffer::kBufferTypeANWBuffer) {
    	// 将 graphic buffer 绑定给 bufferMeta
        status_t err = updateGraphicBufferInMeta_l(
                kPortIndexOutput, omxBuffer.mGraphicBuffer, buffer, header);

        if (err != OK) {
            CLOG_ERROR(fillBuffer, err, FULL_BUFFER(
                    (intptr_t)header->pBuffer, header, fenceFd));
            return err;
        }
    } else if (omxBuffer.mBufferType != OMXBuffer::kBufferTypePreset) {
        return BAD_VALUE;
    }
	// 重置buffer状态
    header->nFilledLen = 0;
    header->nOffset = 0;
    header->nFlags = 0;
	// 等待fence释放
    // meta now owns fenceFd
    status_t res = storeFenceInMeta_l(header, fenceFd, kPortIndexOutput);
    if (res != OK) {
        CLOG_ERROR(fillBuffer::storeFenceInMeta, res, EMPTY_BUFFER(buffer, header, fenceFd));
        return res;
    }

    {
        Mutex::Autolock _l(mDebugLock);
        mOutputBuffersWithCodec.add(header);
        CLOG_BUMPED_BUFFER(fillBuffer, WITH_STATS(EMPTY_BUFFER(buffer, header, fenceFd)));
    }
	// 传递给 OMX 组件
    OMX_ERRORTYPE err = OMX_FillThisBuffer(mHandle, header);
    if (err != OMX_ErrorNone) {
        CLOG_ERROR(fillBuffer, err, EMPTY_BUFFER(buffer, header, fenceFd));
        Mutex::Autolock _l(mDebugLock);
        mOutputBuffersWithCodec.remove(header);
    }
    return StatusFromOMXError(err);
}

• 当ACodec有新的graphic buffer传递下来时，OMXNodeInstance 检查到 buffer type 为 kBufferTypeANWBuffer，会将新的graphic buffer 绑定到 BufferMeta上。
• 等待fence释放完成后就可以让decoder使用了。

status_t OMXNodeInstance::storeFenceInMeta_l(
        OMX_BUFFERHEADERTYPE *header, int fenceFd, OMX_U32 portIndex) {
    // propagate fence if component supports it; wait for it otherwise
    OMX_U32 metaSize = portIndex == kPortIndexInput ? header->nFilledLen : header->nAllocLen;
    if (mMetadataType[portIndex] == kMetadataBufferTypeANWBuffer
            && metaSize >= sizeof(VideoNativeMetadata)) {
        VideoNativeMetadata &nativeMeta = *(VideoNativeMetadata *)(header->pBuffer);
        if (nativeMeta.nFenceFd >= 0) {
            ALOGE("fence (%d) already exists in meta", nativeMeta.nFenceFd);
            if (fenceFd >= 0) {
                ::close(fenceFd);
            }
            return ALREADY_EXISTS;
        }
        nativeMeta.nFenceFd = fenceFd;
    } else if (fenceFd >= 0) {
        CLOG_BUFFER(storeFenceInMeta, "waiting for fence %d", fenceFd);
        sp<Fence> fence = new Fence(fenceFd);
        return fence->wait(IOMX::kFenceTimeoutMs);
    }
    return OK;
}

在等待fence释放之前有个判断：

• 当 mMetadataType 为 kMetadataBufferTypeANWBuffer 时，会判断 BufferHeader 的 nAllocLen 是否大于等于 VideoNativeMetadata 的大小。我这里就盲猜，这是在判断 BufferHeader 是否已经绑定 graphic buffer；
• 当BufferHeader还没有绑定graphic buffer时，会先等待fence释放，这里可能会有阻塞的情况出现，这是不符合预期的，这边不能出现阻塞！