Changeset 3974

Show
Ignore:
Timestamp:
03/23/07 18:11:29 (7 years ago)
Author:
piers
Message:

RTP Payload format for H.264 (RFC 3984)

Most of this work below has been done by Socrates. Tidy up and bug fixes by Piers.

General comments:

1. Implemented the H.264 RTP framing according to RFC 3984. The implementation covers the "Single NAL Unit" (Single NALU packet type) and "Non-Interleaved" modes (STAP-A, FU-A packet types). The interleaved mode (STAP-B, MTAP16, MTAP24, FU-B) has not been implemented at this stage. [I don't know of another tool, closed source or open, that has implemented this mode to date to be able to test with (the current QuickTime? Player 7.1.3 does not implement it either, as far as I know).]

2. The decoder has been tested with the MPEG4IP's mp4live server 1.5.0.1. An example SDP file is provided to set up VIC to decode mp4live streams. Please edit the file default_mpeg4ip.sdp with your details, or better generate your own SDP files using mp4live, and place the edited SDP file in your home directory. The filename should be "default.sdp"

3. The H264 depayloader implementation is currently based libavformat (part of ffpmeg). The plan to is re-write the H264 depayloader under BSD license in the near future.

4. The H.264 VIC encoder will not aggregate NAL units at the moment (STAP-A). The decoder will properly decode such packets.

5. VIC sends SPS/PPS NAL units in-band at the start of H.264 transmission. One has to check whether this can be done periodically. This way we may avoid SDP and sprep-parameter-set.

6. The SDP may be configured for the decoder which uses either a default embedded SDP or from "$HOME/default.sdp"

7. The SDP file is only used to convey SPS/PPS sprop-parameter-sets. IP addresses, ports, etc. are read from VIC command-line as normal.

Location:
vic/branches/mpeg4
Files:
2 added
8 modified

Legend:

Unmodified
Added
Removed
  • vic/branches/mpeg4/Makefile.in

    r3954 r3974  
    148148        codec/ffmpeg_codec.o codec/encoder-mpeg4.o codec/decoder-mpeg4.o \ 
    149149        codec/x264encoder.o codec/encoder-h264.o codec/decoder-h264.o \ 
     150        codec/rtp_h264_depayloader.o \ 
    150151        codec/decoder-h263.o codec/decoder-h263v2.o codec/decoder-cellb.o \ 
    151152        codec/decoder-h261.o codec/decoder-h261v1.o codec/decoder-raw.o \ 
  • vic/branches/mpeg4/codec/databuffer.h

    r3885 r3974  
    1414    bool write(char *, int); 
    1515    bool writeAppend(char *, int); 
     16    char *data; 
    1617 
    1718  private: 
    1819    int length; 
    1920    int size; 
    20     char *data; 
    2121 
    2222}; 
  • vic/branches/mpeg4/codec/decoder-h264.cpp

    r3942 r3974  
    44#include <assert.h> 
    55#include <iostream> 
     6#include <errno.h> 
    67#include "inet.h" 
    78#include "rtp.h" 
     
    1112#include "databuffer.h" 
    1213#include "ffmpeg_codec.h" 
     14#include "rtp_h264_depayloader.h" 
     15 
     16 
     17#define SDP_LINE_LEN 10000 
    1318 
    1419//#define DIRECT_DISPLAY 1 
    15  
    16 using namespace std; 
    17  
    18 extern "C" UCHAR * video_frame; 
     20//using namespace std; 
     21//extern "C" UCHAR * video_frame; 
     22 
    1923 
    2024class H264Decoder:public Decoder 
     
    2428    ~H264Decoder(); 
    2529 
     30    void handleSDP(); 
    2631    virtual void recv(pktbuf *); 
    2732    int colorhist(u_int * hist) const; 
     
    3136 
    3237    /* packet statistics */ 
    33     u_int16_t last_seq;         /* sequence number */ 
     38    uint16_t last_seq;          /* sequence number */ 
    3439 
    3540    UCHAR bitstream[MAX_CODED_SIZE];    /* bitstream data */ 
    3641 
    3742    /* collecting data for a frame */ 
    38     int idx; 
     43    uint16_t idx; 
    3944    int last_mbit; 
    4045    int last_iframe; 
     
    4449    UCHAR xxx_frame[MAX_FRAME_SIZE]; 
    4550    FFMpegCodec h264; 
    46     PacketBuffer *stream; 
    47  
     51    PacketBuffer *stream; //SV: probably this is going to be substituted by the AVPacket->data when we call decode()... 
     52    H264Depayloader *h264depayloader; 
    4853 
    4954    //For DEBUG 
    50     FILE *fptr; 
     55    //FILE *fptr; 
     56    FILE *sdp_fptr; 
    5157}; 
    5258 
     
    6874 
    6975 
    70 H264Decoder::H264Decoder():Decoder(2) 
     76H264Decoder::H264Decoder():Decoder(0 /* 0 byte extra header */) 
    7177{                               /* , codec_(0), */ 
    7278 
     79 
     80    //Barz: ============================================= 
    7381    decimation_ = 411; 
    7482    /* 
     
    7987    inh_ = 0; 
    8088 
    81      /*XXX*/ resize(inw_, inh_); 
     89     /*XXX*/  
     90    resize(inw_, inh_); 
    8291 
    8392    // libavcodec 
     
    8998    last_iframe = 0; 
    9099    last_seq = 0; 
     100    //=================================================== 
     101 
     102 
     103    //SV: =============================================== 
     104    //Create payloader 
     105    h264depayloader = new H264Depayloader; 
     106 
     107    handleSDP(); 
     108    //make sure all codec params are set up properly before decoding 
     109 
     110    //check whether AVPacket struct can be used in stead of Barz's PacketBuffer 
     111    //=================================================== 
     112 
    91113 
    92114    //256 packets, each 1600 byte (default will not exceed 1600 byte) 
    93115    //cout << "new PacketBuffer..\n"; 
    94     stream = new PacketBuffer(1024, 1600); 
     116    stream = new PacketBuffer(1024, 1600); //SV: 1024 = ???  
    95117    startPkt = false; 
     118} 
     119 
     120void 
     121H264Decoder::handleSDP() 
     122{ 
     123    char SdpFilename[SDP_LINE_LEN]; 
     124    char *line, *sdp_string; 
     125    char *SdpLine=NULL; 
     126    int n_char; 
     127    size_t nBytes = 0; 
     128    ssize_t SdpRead; 
     129    char defaultSDP[]="a=rtpmap:96 H264/90000\na=fmtp:96 profile-level-id=00000d; packetization-mode=1\n"; 
     130 
     131    sprintf(SdpFilename, "%s/default.sdp", getenv("HOME")); 
    96132 
    97133    //fptr = fopen("out.m4v", "w"); 
     134    if ((sdp_fptr = fopen(SdpFilename, "r")) != NULL ) { 
     135        //fprintf(stderr, "H264_RTP: Opened SDP file %s for read.\n", SdpFilename); 
     136 
     137      //Read SDP file into struct 
     138      //fprintf(stderr, "H264_RTP: Spitting SDP ================================================\n"); 
     139      while ((SdpRead = getline(&SdpLine, &nBytes, sdp_fptr)) != -1) { 
     140        //fprintf(stderr, "H264_RTP: Read %d bytes from SDP file.\n", SdpRead); 
     141        //fprintf(stderr, "%s", SdpLine); 
     142        //call SDP parse h264 routine 
     143          h264depayloader->parse_h264_sdp_line(h264.c, h264depayloader->h264_extradata, SdpLine); 
     144      } 
     145      if (SdpLine) 
     146          free(SdpLine); 
     147         
     148    } else { 
     149        fprintf(stderr, "H264_RTP: Couldn't open SDP file %s to read. Errno = %d\n", SdpFilename, errno); 
     150        fprintf(stderr, "H264_RTP: Using default SDP: %s \n", defaultSDP); 
     151 
     152      line=defaultSDP; 
     153      do { 
     154        n_char = strcspn(line, "\n"); 
     155        SdpLine = (char *)malloc(n_char+1); 
     156        memset(SdpLine, '\0', n_char+1); 
     157        strncpy(SdpLine, line, n_char); 
     158        line += n_char + 1; 
     159          h264depayloader->parse_h264_sdp_line(h264.c, h264depayloader->h264_extradata, SdpLine); 
     160        free(SdpLine); 
     161      } while (n_char != 0); 
     162    //fprintf(stderr, "H264_RTP: Done spitting SDP ===========================================\n"); 
     163    } 
    98164} 
    99165 
     
    101167{ 
    102168    delete stream; 
     169    delete h264depayloader; 
    103170    //fclose(fptr); 
     171    if (sdp_fptr != NULL) fclose(sdp_fptr); 
     172    //fprintf(stderr, "H264_RTP: Closed SDP file.\n"); 
    104173} 
    105174 
    106175int H264Decoder::colorhist(u_int * hist)  const 
    107176{ 
     177    UNUSED(hist); 
     178 
    108179    return (1); 
    109180} 
     
    115186    u_char *bp = pb->dp + hdrsize; 
    116187    int cc = pb->len - hdrsize; 
    117     static int iframe_c = 0, pframe_c = 0; 
     188    //static int iframe_c = 0, pframe_c = 0; 
    118189 
    119190    int mbit = ntohs(rh->rh_flags) >> 7 & 1; 
    120     int seq = ntohs(rh->rh_seqno); 
     191    uint16_t seq = ntohs(rh->rh_seqno); 
    121192    int ts = ntohl(rh->rh_ts); 
    122193 
     194 
     195 
     196    //Barz: ============================================= 
    123197    if (!startPkt) { 
    124198       stream->clear(); 
    125199       startPkt = true; 
    126200       idx = seq; 
    127            last_seq = seq - 1; 
     201       last_seq = seq - 1; 
    128202    } 
    129203           
     
    134208 
    135209    if (abs(seq - last_seq) > 5) { 
    136             debug_msg("h264dec: sequece interrupt!\n"); 
     210            //fprintf(stderr, "H264_RTP: sequece interrupt!\n"); 
    137211            idx = seq; 
    138212            pktIdx = 0; 
    139213            stream->clear(); 
    140214    }else if (last_seq + 1 != seq) { 
    141             /* oops - missing packet */ 
    142             debug_msg("h264dec: missing packet\n"); 
    143     } 
     215            // oops - missing packet 
     216            //fprintf(stderr, "H264_RTP: missing packet\n"); 
     217    } 
     218 
     219    //=================================================== 
     220 
    144221 
    145222    //copy packet 
    146     stream->write(pktIdx, cc, (char *) bp); 
    147      
     223    //stream->write(pktIdx, cc, (char *) bp); 
     224    //fprintf(stderr, "H264_RTP: -------------------------------- seq = %d, m=%d, ts=%d, cc=%d\n", seq, mbit, ts, cc); 
     225    //fprintf(stderr, "H264_RTP: pktIdx = %d\n", pktIdx); 
     226    int yo = h264depayloader->h264_handle_packet(h264depayloader->h264_extradata, pktIdx, stream, /*ts,*/ bp, cc); 
     227    //fprintf(stderr, "H264_RTP: h264_handle_packet = %d\n", yo); 
     228 
     229 
     230    //Barz: ============================================= 
     231 
    148232    last_seq = seq; 
    149233    int len=0; 
    150          
    151234    if (mbit) { 
    152235            stream->setTotalPkts(pktIdx + 1); 
    153236 
    154             DataBuffer *f;           
     237            DataBuffer *f; 
    155238            if (stream->isComplete()) { 
    156                         f = stream->getStream(); 
     239                    f = stream->getStream(); 
    157240                    len =  h264.decode((UCHAR *) f->getData(), f->getDataSize(), xxx_frame); 
    158241            } 
    159242             
    160243            if (len < 0) { 
    161                debug_msg("h264dec: frame error\n"); 
     244                //fprintf(stderr, "H264_RTP: frame error\n"); 
    162245            } 
    163246            
     
    170253                        Decoder::redraw(xxx_frame); 
    171254            } 
    172         stream->clear(); 
     255            stream->clear(); 
    173256            idx = seq+1; 
    174                       
    175     } 
     257    } 
     258 
     259    //=================================================== 
     260 
    176261    pb->release(); 
    177262} 
  • vic/branches/mpeg4/codec/encoder-h264.cpp

    r3956 r3974  
    1818#include "x264encoder.h" 
    1919#include "deinterlace.h" 
     20/*extern "C" 
     21{ 
     22#include "base64.h" 
     23}*/ 
    2024 
    2125static Transmitter *tx; 
     
    120124    pktbuf *pb; 
    121125    rtphdr *rh; 
    122     //int n,ps, len; 
    123126    ts = vf->ts_; 
    124127    tx = tx_; 
    125128    pool = pool_; 
    126     bool first = true; 
    127  
    128     int i_nal, i, sent_size; 
     129 
     130    int numNAL, i, sent_size = 0; 
    129131    int frame_size = 0; 
     132    unsigned char f_seq = 0; 
     133    //unsigned char f_total_pkt = 0; 
     134    int RTP_HDR_LEN = sizeof(rtphdr); 
     135    int NAL_FRAG_THRESH = tx->mtu() - RTP_HDR_LEN; /* payload max in one packet */ 
     136    //fprintf(stderr, "MTU=%d, RTP_HDR_LEN=%d\n", NAL_FRAG_THRESH, RTP_HDR_LEN); 
     137 
    130138 
    131139    tx->flush(); 
     
    141149 
    142150    frame_size = vf->width_ * vf->height_; 
    143  
    144151    deinterlacer.render(vf->bp_, vf->width_, vf->height_); 
    145          
    146152    enc->encodeFrame(vf->bp_); 
    147  
    148     i_nal = enc->numNAL(); 
    149  
    150     sent_size = 0; 
    151     unsigned char f_seq = 0; 
    152     unsigned char f_total_pkt = 0; 
    153     //TODO: send out i_nal packets 
    154     for (i = 0; i < i_nal; i++) { 
     153    numNAL = enc->numNAL(); 
     154 
     155    //Send out numNAL packets framed according to RFC3984 
     156    for (i = 0; i < numNAL; i++) { 
     157 
     158        bool firstFragment = true; 
     159        int FU_HDR_LEN = 1; 
     160        int offset = 0; 
    155161 
    156162        enc->getNALPacket(i, fOut); 
    157  
    158         sent_size += fOut->getDataSize(); 
     163        int nalSize1 = fOut->getDataSize(); 
     164        int nalSize = nalSize1-5; 
     165        char *data1 = fOut->getData(); 
    159166        char *data = fOut->getData(); 
    160  
    161  
    162         int nalSize = fOut->getDataSize(); 
    163         if (i == i_nal - 1 && f_total_pkt == 0) 
    164             f_total_pkt = f_seq + (nalSize / 1000) + 2; 
    165         int offset = 0; 
     167        uint8_t NALhdr = data1[4]; //SV-XXX why does our x.264 provide 4-byte StartSync in the NALU? 
     168        uint8_t NALtype = NALhdr & 0x1f; 
     169        //fprintf(stderr, "Got NALhdr=0x%02x, NALtype=0x%02x from encoded frame.\n", NALhdr, NALtype); 
     170        memcpy(data, &data1[5], nalSize); 
     171 
     172        sent_size += nalSize; 
     173 
     174 
    166175        while (nalSize > 0) { 
    167             pb = pool_->alloc(vf->ts_, RTP_PT_H264); 
     176            pb = pool->alloc(vf->ts_, RTP_PT_H264); 
    168177            rh = (rtphdr *) pb->data; 
    169             if (nalSize > 1000) { 
    170                 memcpy(&pb->data[14], data + offset, 1000); 
    171  
    172                 pb->len = 1000 + 14; 
    173                 offset += 1000; 
    174                 nalSize -= 1000; 
     178 
     179            if (nalSize <= NAL_FRAG_THRESH) { 
     180                //============================================== 
     181                //Single NAL or last fragment of FU-A 
     182                //============================================== 
     183 
     184                rh->rh_flags |= htons(RTP_M);   // set M bit 
     185                pb->len = nalSize + RTP_HDR_LEN + FU_HDR_LEN; 
     186 
     187                fprintf(stderr, "NAL : "); 
     188 
     189                if (FU_HDR_LEN==2) { 
     190                        //============================================== 
     191                        //Last fragment of FU-A 
     192                        //============================================== 
     193                        pb->data[12] = 0x00 | (NALhdr & 0x60) | 28;     //FU indicator 
     194                        pb->data[13] = 0x40  | NALtype;                 //FU header 
     195                         
     196                        fprintf(stderr, "FU_Indicator=0x%02x, FU_Header=0x%02x, ", pb->data[12], pb->data[13]); 
     197                }  
     198                else { 
     199                        pb->data[12] = NALhdr;                          //NAL Header 
     200                        fprintf(stderr, "-----------------, --------------, "); 
     201                } 
     202 
     203                memcpy(&pb->data[RTP_HDR_LEN + FU_HDR_LEN], data + offset, nalSize); 
     204 
     205                fprintf(stderr, "i=%d/%d, nalSize=%4d len=%4d firstFrag=%d offset=%4d\n", i, numNAL, nalSize, pb->len, firstFragment, offset); 
     206 
     207                nalSize = 0; 
     208                offset = 0; 
     209 
     210            } else { 
     211                //============================================== 
     212                //FU-A (not the last fragment though) 
     213                //============================================== 
     214 
     215                FU_HDR_LEN = 2; 
     216                pb->len = (NAL_FRAG_THRESH - FU_HDR_LEN) + RTP_HDR_LEN + FU_HDR_LEN; 
     217 
     218                pb->data[12] = 0x00 | (NALhdr & 0x60) | 28;                     //FU indicator 
     219                pb->data[13] = ( (firstFragment) ? 0x80 : 0x00 ) | NALtype;     //FU header 
     220 
     221                memcpy(&pb->data[RTP_HDR_LEN + FU_HDR_LEN], data + offset, NAL_FRAG_THRESH - FU_HDR_LEN); 
     222 
     223                fprintf(stderr, "FU-A: FU_Indicator=0x%02x, FU_Header=0x%02x, i=%d/%d, nalSize=%4d len=%4d firstFrag=%d offset=%4d\n",  pb->data[12], pb->data[13], i, numNAL, nalSize, pb->len, firstFragment, offset); 
     224 
     225                nalSize -= (NAL_FRAG_THRESH-FU_HDR_LEN); 
     226                offset += (NAL_FRAG_THRESH-FU_HDR_LEN); 
     227                firstFragment = false; 
    175228            } 
    176             else { 
    177                 if (i == i_nal - 1) { 
    178                     //Last Packet 
    179                     rh->rh_flags |= htons(RTP_M);       // set M bit 
    180                 } 
    181                 memcpy(&pb->data[14], data + offset, nalSize); 
    182  
    183                 pb->len = nalSize + 14; 
    184                 nalSize = 0; 
    185             } 
    186             //printf("send out %d\n", pb->len); 
     229 
    187230            tx->send(pb); 
    188231            f_seq++; 
    189232        } 
    190     } 
     233 
     234    } 
     235 
    191236    frame_seq++; 
    192237 
     238 
    193239    return (kbps*1024) / (fps*8); 
    194240} 
  • vic/branches/mpeg4/codec/packetbuffer.cpp

    r3942 r3974  
    2929} 
    3030 
     31void PacketBuffer::writeAppend(int idx, int size, char *buffer) 
     32{ 
     33    if (idx > MAX_PACKETS || idx < 0) 
     34        return; 
     35    isDataRecv[idx] = packets[idx]->writeAppend(buffer, size); 
     36} 
     37 
    3138void PacketBuffer::write(int idx, int size, char *buffer) 
    3239{ 
     
    3441        return; 
    3542    isDataRecv[idx] = packets[idx]->write(buffer, size); 
     43} 
     44 
     45void PacketBuffer::setSize(int idx, int size) 
     46{ 
     47    if (idx > MAX_PACKETS || idx < 0) 
     48        return; 
     49    isDataRecv[idx] = packets[idx]->setSize(size); 
    3650} 
    3751 
     
    4256    for (int i = 0; i < totalPkts; i++){         
    4357        if (!isDataRecv[i]){ 
    44             debug_msg("lost packet %d\n", i);    
     58            //debug_msg("lost packet %d\n", i);  
    4559            return false; 
    4660        } 
  • vic/branches/mpeg4/codec/packetbuffer.h

    r3885 r3974  
    44#define MAX_PACKETS 2048 
    55 
     6#include "databuffer.h" 
    67 
    7 class DataBuffer; 
     8 
     9//class DataBuffer; 
    810class PacketBuffer 
    911{ 
     
    1214       ~PacketBuffer(); 
    1315    void write(int, int, char *);       //pktIdx, length, buffer 
     16    void writeAppend(int, int, char *); //pktIdx, length, buffer 
     17    void setSize(int, int);     //pktIdx, length 
    1418    void setTotalPkts(int); 
    1519    int getTotalPkts(); 
     
    1721    DataBuffer *getStream(); 
    1822    void clear(); 
     23    DataBuffer *packets[MAX_PACKETS]; 
    1924  private: 
    2025    int totalPkts; 
     
    2227    int maxLen; 
    2328    DataBuffer *stream; 
    24     DataBuffer *packets[MAX_PACKETS]; 
    2529    bool isDataRecv[MAX_PACKETS]; 
    2630}; 
  • vic/branches/mpeg4/rtp/rtp.h

    r3885 r3974  
    6262#define RTP_PT_DV               123     /* DV */ 
    6363#define RTP_PT_MPEG4            45 
    64 #define RTP_PT_H264             47 
     64#define RTP_PT_H264             96 
    6565 
    6666/* backward compat hack for decoding RTPv1 ivs streams */ 
  • vic/branches/mpeg4/tcl/cf-main.tcl

    r3885 r3974  
    5050set rtp_type(123) dv 
    5151set rtp_type(45) mpeg4 
    52 set rtp_type(47) h264 
     52set rtp_type(96) h264 
    5353 
    5454