Android 4.0 Ice Cream Sandwich Media Framework (5)

Continued (5)

---===NuPlayer===---


In part 4, HTTPLiveSource start downloading the m3u8 file and the successive media segments

Back to NuPlayer

In Media Framework  :android_src\framework\base\media\libmediaplayerservice\nuplayer\NuPlayer.cpp

void NuPlayer::onMessageReceived(const sp &msg) {
    switch (msg->what()) {
        case kWhatStart:
        {
            LOGV("kWhatStart");


            mVideoIsAVC = false;
            mAudioEOS = false;
            mVideoEOS = false;
            mSkipRenderingAudioUntilMediaTimeUs = -1;
            mSkipRenderingVideoUntilMediaTimeUs = -1;
            mVideoLateByUs = 0;
            mNumFramesTotal = 0;
            mNumFramesDropped = 0;


            mSource->start();//HTTPLiveSource


            mRenderer = new Renderer(
                    mAudioSink,
                    new AMessage(kWhatRendererNotify, id()));


            looper()->registerHandler(mRenderer);


            postScanSources();
            break;
        }
    }
}

In Media Framework  :android_src\framework\base\media\libmediaplayerservice\nuplayer\NuPlayerRenderer.cpp


NuPlayer::Renderer::Renderer(
        const sp &sink,
        const sp &notify)
    : mAudioSink(sink),
      mNotify(notify),
      mNumFramesWritten(0),
      mDrainAudioQueuePending(false),
      mDrainVideoQueuePending(false),
      mAudioQueueGeneration(0),
      mVideoQueueGeneration(0),
      mAnchorTimeMediaUs(-1),
      mAnchorTimeRealUs(-1),
      mFlushingAudio(false),
      mFlushingVideo(false),
      mHasAudio(false),
      mHasVideo(false),
      mSyncQueues(false),
      mPaused(false),
      mLastPositionUpdateUs(-1ll),
      mVideoLateByUs(0ll) {
}


Note that NuPlayer pass a message "kWhatRendererNotify" to construct the NuPlayerRenderer, NuPlayerRenderer can use this message(stored as mNotify) to notify NuPlayer some information in the future


In Media Framework  :android_src\framework\base\media\libmediaplayerservice\nuplayer\NuPlayer.cpp

void NuPlayer::postScanSources() {
    if (mScanSourcesPending) {
        return;
    }


    sp msg = new AMessage(kWhatScanSources, id());
    msg->setInt32("generation", mScanSourcesGeneration);
    msg->post();


    mScanSourcesPending = true;
}

"mScanSourcesPending=true" means the kWhatScanSources message is pending in the Looper's message queue.

void NuPlayer::onMessageReceived(const sp &msg) {
    switch (msg->what()) {
        case kWhatScanSources:
        {
            mScanSourcesPending = false;


            instantiateDecoder(false, &mVideoDecoder);


            if (mAudioSink != NULL) {
                instantiateDecoder(true, &mAudioDecoder);
            }


            status_t err;
            if ((err = mSource->feedMoreTSData()) != OK) {
                if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
                    // We're not currently decoding anything (no audio or
                    // video tracks found) and we just ran out of input data.


                    if (err == ERROR_END_OF_STREAM) {
                        notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
                    } else {
                        notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
                    }
                }
                break;
            }


            if (mAudioDecoder == NULL || mVideoDecoder == NULL) {
                msg->post(100000ll);
                mScanSourcesPending = true;
            }
            break;
        }
    }
}

status_t NuPlayer::instantiateDecoder(bool audio, sp *decoder) {
    if (*decoder != NULL) {
        return OK;
    }


    sp meta = mSource->getFormat(audio);


    if (meta == NULL) {
        return -EWOULDBLOCK;
    }
...
...
...
}


Note that mSource here refer to HTTPLiveSource.

At first, the meta == NULL and decoder == NULL because the media segment has not been downloaded or been parsed. (none of the buffer stored in mDataSource).


In Media Framework  :android_src\framework\base\media\libmediaplayerservice\nuplayer\HTTPLiveSource.cpp

status_t NuPlayer::HTTPLiveSource::feedMoreTSData() {


    sp source =
        static_cast(mLiveSession->getDataSource().get());


    for (int32_t i = 0; i < 50; ++i) {
        char buffer[188];
        ssize_t n = source->readAtNonBlocking(mOffset, buffer, sizeof(buffer));


        if (n == -EWOULDBLOCK) {
            break;
        }
...
...
    }


    return OK;
}


Because of the same reason above, (n == -EWOULDBLOCK) is true and break the for loop.

Therefore NuPlayer will continuously do msg->post(100000ll) to post kWhatScanSources message.

When the meta in instantiateDecoder not equal to NULL ?

Back to see what the LiveSession do when onDownloadNext() executed

In Media Framework  :android_src\framework\base\media\libstagefright\httplive\LiveSession.cpp

void LiveSession::onDownloadNext() {
...
...
...
    mDataSource->queueBuffer(buffer);
    postMonitorQueue();
}

The buffers will be stored in mDataSource's buffer queue

Let's see feedMoreTSData() in HTTPLiveSource again

In Media Framework  :android_src\framework\base\media\libmediaplayerservice\nuplayer\HTTPLiveSource.cpp

status_t NuPlayer::HTTPLiveSource::feedMoreTSData() {


    sp source =
        static_cast(mLiveSession->getDataSource().get());


    for (int32_t i = 0; i < 50; ++i) {
        char buffer[188];
        ssize_t n = source->readAtNonBlocking(mOffset, buffer, sizeof(buffer));




        if (n == -EWOULDBLOCK) {
            break;
        }


        status_t err = mTSParser->feedTSPacket(buffer, sizeof(buffer));


        mOffset += n;
    }
    return OK;
}

If there are buffer in the LiveDataSource's buffer queue, then mTSParser->feedTSPacket() will be called


In Media Framework  :android_src\framework\base\media\libstagefright\mpegts\ATSParser.cpp

status_t ATSParser::feedTSPacket(const void *data, size_t size) {
    CHECK_EQ(size, kTSPacketSize);


    ABitReader br((const uint8_t *)data, kTSPacketSize);
    return parseTS(&br);
}


status_t ATSParser::parseTS(ABitReader *br) {


    unsigned sync_byte = br->getBits(8);
    CHECK_EQ(sync_byte, 0x47u);
...
...
    return OK;
}


The parseTS() involves the format and specification of TS(transport stream),

and the process of parseTS() will end with parsePES()


status_t ATSParser::Stream::parsePES(ABitReader *br) {
...
...
            onPayloadData(
                    PTS_DTS_flags, PTS, DTS, br->data(), dataLength);
}

void ATSParser::Stream::onPayloadData(
        unsigned PTS_DTS_flags, uint64_t PTS, uint64_t DTS,
        const uint8_t *data, size_t size) {
    LOGV("onPayloadData mStreamType=0x%02x", mStreamType);


    int64_t timeUs = 0ll;  // no presentation timestamp available.
    if (PTS_DTS_flags == 2 || PTS_DTS_flags == 3) {
        timeUs = mProgram->convertPTSToTimestamp(PTS);
    }


    status_t err = mQueue->appendData(data, size, timeUs);


    sp accessUnit;
    while ((accessUnit = mQueue->dequeueAccessUnit()) != NULL) {
        if (mSource == NULL) {
            sp meta = mQueue->getFormat();


            if (meta != NULL) {
                LOGV("Stream PID 0x%08x of type 0x%02x now has data.",
                     mElementaryPID, mStreamType);


                mSource = new AnotherPacketSource(meta);
                mSource->queueAccessUnit(accessUnit);
            }
        } else if (mQueue->getFormat() != NULL) {
            // After a discontinuity we invalidate the queue's format
            // and won't enqueue any access units to the source until
            // the queue has reestablished the new format.


            if (mSource->getFormat() == NULL) {
                mSource->setFormat(mQueue->getFormat());
            }
            mSource->queueAccessUnit(accessUnit);
        }
    }
}

(Here mQueue refer to ESQueue in android_src\framework\base\media\libstagefright\mpegts\ESQueue.cpp)

The onPayloadData finally put the H264 AccessUnit into mSource(refer to AnotherPacketSource)

and set the format of mSource

Back to NuPlayer

In Media Framework  :android_src\framework\base\media\libmediaplayerservice\nuplayer\NuPlayer.cpp


status_t NuPlayer::instantiateDecoder(bool audio, sp *decoder) {
    if (*decoder != NULL) {
        return OK;
    }


    sp meta = mSource->getFormat(audio);


    if (meta == NULL) {
        return -EWOULDBLOCK;
    }


    sp notify =
        new AMessage(audio ? kWhatAudioNotify : kWhatVideoNotify,
                     id());


    *decoder = audio ? new Decoder(notify) :
                       new Decoder(notify, mNativeWindow);
    looper()->registerHandler(*decoder);


    (*decoder)->configure(meta);


    return OK;
}

Take a look of mSource->getFormat()

In Media Framework  :android_src\framework\base\media\libmediaplayerservice\nuplayer\HTTPLiveSource.cpp

sp NuPlayer::HTTPLiveSource::getFormat(bool audio) {

    ATSParser::SourceType type =

        audio ? ATSParser::AUDIO : ATSParser::VIDEO;

    sp source =

        static_cast(mTSParser->getSource(type).get());

    if (source == NULL) {

        return NULL;

    }

    return source->getFormat();

}

We can get the none-null format if the process of parsePES is finished

It's time to new Decoder, note that NuPlayer also pass a message kWhatVideoNotify (if video) to construct the Decoder.

In Media Framework  :android_src\framework\base\media\libmediaplayerservice\nuplayer\NuPlayerDecoder.cpp

NuPlayer::Decoder::Decoder(

        const sp &notify,

        const sp &nativeWindow)

    : mNotify(notify),

      mNativeWindow(nativeWindow) {

}

void NuPlayer::Decoder::configure(const sp &meta) {

    sp notifyMsg =

        new AMessage(kWhatCodecNotify, id());

    bool needDedicatedLooper = !strncasecmp(mime, "video/", 6);

    mCodec = new ACodec;

    if (needDedicatedLooper && mCodecLooper == NULL) {

        mCodecLooper = new ALooper;

        mCodecLooper->setName("NuPlayerDecoder");

        mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);

    }

    (needDedicatedLooper ? mCodecLooper : looper())->registerHandler(mCodec);

    mCodec->setNotificationMessage(notifyMsg);

    mCodec->initiateSetup(format);

}

A message "kWhatCodecNotify" is passed to ACodec for notify back. The role of ACodec is like OMXCodec in Stagefright

In Media Framework  :android_src\framework\base\media\libstagefright\ACodec.cpp 

ACodec::ACodec()

    : mNode(NULL),

      mSentFormat(false) {

    mUninitializedState = new UninitializedState(this);

    mLoadedToIdleState = new LoadedToIdleState(this);

    mIdleToExecutingState = new IdleToExecutingState(this);

    mExecutingState = new ExecutingState(this);

    mOutputPortSettingsChangedState =new OutputPortSettingsChangedState(this);

    mExecutingToIdleState = new ExecutingToIdleState(this);

    mIdleToLoadedState = new IdleToLoadedState(this);

    mFlushingState = new FlushingState(this);

    mPortEOS[kPortIndexInput] = mPortEOS[kPortIndexOutput] = false;

    mInputEOSResult = OK;

    changeState(mUninitializedState);

}

ACodec inherit AHierarchicalStateMachine, and implement the state machine.

We can see the code above with the states: 

==================================

UninitializedState

LoadedToIdleState

IdleToExecutingState

ExecutingState 

OutputPortSettingsChangedState 

ExecutingToIdleState 

IdleToLoadedState 

FlushingState 

================================== 

In Media Framework  :android_src\framework\base\media\libstagefright\foundation\AHierarchicalStateMachine.cpp

void AHierarchicalStateMachine::changeState(const sp &state) {

...

...

    for (size_t i = 0; i < A.size(); ++i) {

        A.editItemAt(i)->stateExited();

    }

    for (size_t i = B.size(); i-- > 0;) {

        B.editItemAt(i)->stateEntered();

    }

}

Each state except UninitializedState in ACodec override the stateEntered, but stateExited() not.

In Media Framework  :android_src\framework\base\media\libstagefright\ACodec.cpp  

void ACodec::setNotificationMessage(const sp &msg) {

    mNotify = msg;

}

void ACodec::initiateSetup(const sp &msg) {

    msg->setWhat(kWhatSetup);

    msg->setTarget(id());

    msg->post();

}

To be continued.