H264 SIP Intercom videophone, baresip+GStreamer+OMX - Raspberry Pi Forums

hi all,

have made few dirty code changes in baresip allow hw h264 encoding raspberry pi usb webcam. code changes made allow 1-way video.
work pretty fine linphone on mac os, delays of video happening when resolution gets above 320x240 24fps. wonder why, pure gstreamer can stream 640x480@24fps without issues. cpu @~50%
shed light on how improve it?

here way set up:
0. setup raspberry b1.2 512mb
https://www.raspberrypi.org/documentati ... ng-images/

install lite image on 4-8gb sd card

raspi-config ; extend root partition

apt-get update

apt-get install asterisk

apt-get install gstreamer1.0
apt-get install libasound2-dev


1. mac linphone:
https://www.linphone.org/snapshots/maco ... latest.pkg

2. asterisk on raspberry pi:
apt-get install asterisk

/etc/asterisk/sip.conf
[general]
transport=udp
videosupport=yes

[friends_internal](!)
type=friend
host=dynamic
context=from-internal
allow=all

[bare](friends_internal)
secret=bare;

[nowy](friends_internal)
secret=nowy;


## append /etc/asterisk/extensions.conf
[from-internal]
exten=>9999,1,dial(sip/bare,20)
exten=>5555,1,dial(sip/nowy,20)


3. small changes baresip + compile
wget http://www.creytiv.com/pub/re-0.4.15.tar.gz
wget http://www.creytiv.com/pub/rem-0.4.7.tar.gz

make make install , dont forget to:

sudo ldconfig


wget http://www.creytiv.com/pub/baresip-0.4.17.tar.gz
unpack
changes:
/home/pi/baresip/modules/gst_video1/encode.c

>>>>cut<<<

static int pipeline_init(struct videnc_state *st, const struct vidsz *size)
{
gstappsrc *source;
gstappsink *sink;
gstbus *bus;
gerror* gerror = null;
char pipeline[1024];
char mystring[1024];
gststatechangereturn ret;
int err = 0;

// tom

file * pfile;

pfile = fopen ("/pipe.txt" , "r");
if (pfile == null) perror ("error opening file");
else {
if ( fgets (mystring , 500 , pfile) != null )
puts (mystring);
fclose (pfile);
}


if (!st || !size)
return einval;

snprintf(pipeline, sizeof(pipeline),mystring,
size->w, size->h,
st->encoder.fps);//, st->encoder.bitrate / 1000 /* kbit/s */);


puts("\n!!!!!!\n");

puts(pipeline);

puts("\n!!!!!!\n");


/* initialize pipeline. */
st->streamer.pipeline = gst_parse_launch(pipeline, &gerror);


>>>>cut<<<<

/home/pi/baresip/src/call.c

>>>>cut<<<<

#ifdef use_video
/* video stream */
sc = sdp_media_rformat(stream_sdpmedia(video_strm(call->video)), null);
if (sc) {
err = video_encoder_set(call->video, sc->data, sc->pt,
sc->params);
//err |= video_decoder_set(call->video, sc->data, sc->pt,
// sc->rparams);
if (!err) {
err = video_start(call->video, call->peer_uri);
}
if (err) {
warning("call: video stream error: %m\n", err);
}
}
else if (call->video) {
info("call: video stream disabled..\n");
}

if (call->bfcp) {
err = bfcp_start(call->bfcp);
if (err) {
warning("call: not start bfcp: %m\n", err);
}
}
#endif

>>>>cut<<<<

/home/pi/baresip/src/video.c

>>>>cut<<<<
static int video_stream_decode(struct vrx *vrx, const struct rtp_header *hdr,
struct mbuf *mb)
{
struct video *v = vrx->video;
struct vidframe frame;
struct le *le;
int err = 0;

if (!hdr || !mbuf_get_left(mb))
return 0;

lock_write_get(vrx->lock);

/* no decoder set */
if (!vrx->dec) {
// warning("video: no video decoder!\n");
goto out;
}

frame.data[0] = null;
>>>>cut<<<<
int video_decoder_set(struct video *v, struct vidcodec *vc, int pt_rx,
const char *fmtp)
{
struct vrx *vrx;
int err = 0;

if (!v)
return einval;

/* handle vidcodecs without decoder */
if (!vc->decupdh) {
struct vidcodec *vcd;

// info("video: vidcodec '%s' has no decoder\n", vc->name);

vcd = (struct vidcodec *)vidcodec_find_decoder(vc->name);
if (!vcd) {
// warning("video: not find decoder (%s)\n",
// vc->name);
return enoent;
}

vc = vcd;
}

vrx = &v->vrx;

>>>>cut<<<<

make
make install

root@raspberrypi:/home/pi# cat /pipe.txt
appsrc name=source is-live=true block=true do-timestamp=true max-bytes=1000000 ! videoparse width=%d height=%d format=i420 framerate=%d/1 ! decodebin ! videoconvert ! omxh264enc ! appsink name=sink emit-signals=true drop=true

4. baresip config

root@raspberrypi:/home/pi/.baresip# cat config
#
# baresip configuration
#

#------------------------------------------------------------------------------

# core
poll_method epoll # poll, select, epoll ..

# sip
sip_trans_bsize 128
#sip_listen 0.0.0.0:5060
#sip_certificate cert.pem

# audio
audio_player alsa,default
audio_source alsa,default
audio_alert alsa,default
audio_srate 8000-48000
audio_channels 1-2
#ausrc_srate 48000
#auplay_srate 48000
#ausrc_channels 0
#auplay_channels 0

# video
#video_source v4l2,/dev/video0
#video_display x11,nil
video_size 3200x240
video_bitrate 1000000000
video_fps 24

# avt - audio/video transport
rtp_tos 184
#rtp_ports 10000-20000
#rtp_bandwidth 512-1024 # [kbit/s]
rtcp_enable yes
rtcp_mux no
jitter_buffer_delay 5-10 # frames
rtp_stats no

# network
#dns_server 10.0.0.1:53
#net_interface eth0

# bfcp
#bfcp_proto udp

#------------------------------------------------------------------------------
# modules

module_path /usr/local/lib/baresip/modules

# ui modules
module stdio.so
#module cons.so
#module evdev.so
#module httpd.so

# audio codec modules (in order)
#module opus.so
#module silk.so
#module amr.so
#module g7221.so
#module g722.so
#module g726.so
module g711.so
#module gsm.so
#module l16.so
#module speex.so
#module bv32.so

# audio filter modules (in encoding order)
module vumeter.so
#module sndfile.so
#module speex_aec.so
#module speex_pp.so
#module plc.so

# audio driver modules
module alsa.so
#module portaudio.so
#module aubridge.so

# video codec modules (in order)
module gst_video1.so
#module avcodec.so
#module vpx.so

# video filter modules (in encoding order)
#module selfview.so

# video source modules
#module v4l.so
module v4l2.so
#module x11grab.so
#module cairo.so


# video display modules
#module x11.so
#module sdl2.so

# audio/video source modules
#module rst.so
#module gst.so

# media nat modules
module stun.so
module turn.so
module ice.so
#module natpmp.so

# media encryption modules
#module srtp.so
#module dtls_srtp.so


#------------------------------------------------------------------------------
# temporary modules (loaded unloaded)

module_tmp uuid.so
module_tmp account.so


#------------------------------------------------------------------------------
# application modules

module_app auloop.so
module_app contact.so
module_app menu.so
#module_app mwi.so
#module_app natbd.so
#module_app presence.so
#module_app syslog.so
module_app vidloop.so


#------------------------------------------------------------------------------
# module parameters


cons_listen 0.0.0.0:5555

http_listen 0.0.0.0:8000

evdev_device /dev/input/event0

# speex codec parameters
speex_quality 7 # 0-10
speex_complexity 7 # 0-10
speex_enhancement 0 # 0-1
speex_mode_nb 3 # 1-6
speex_mode_wb 6 # 1-6
speex_vbr 0 # variable bit rate 0-1
speex_vad 0 # voice activity detection 0-1
speex_agc_level 8000

# opus codec parameters
opus_bitrate 28000 # 6000-510000

# nat behavior discovery
natbd_server creytiv.com
natbd_interval 600 # in seconds

# selfview
video_selfview window # {window,pip}
#selfview_size 64x64

# ice
ice_turn no
ice_debug no
ice_nomination regular # {regular,aggressive}
ice_mode full # {full,lite}

5. account baresip
root@raspberrypi:/home/pi/.baresip# cat accounts
<sip:bare:bare@192.168.1.66>;video_codecs=h264

6. call baresip linphone (command call letter "d" number 5555

pi@raspberrypi:~ $ whoami
pi
pi@raspberrypi:~ $ baresip
baresip v0.4.17 copyright (c) 2010 - 2016 alfred e. heggestad et al.
local network address: ipv4=eth0:192.168.1.66
aucodec: pcmu/8000/1
aucodec: pcma/8000/1
aufilt: vumeter
ausrc: alsa
auplay: alsa
vidcodec: h264
gst_video: using gstreamer (gstreamer 1.4.4)
vidsrc: v4l2
medianat: stun
medianat: turn
medianat: ice
populated 1 account
populated 3 contacts
populated 2 audio codecs
populated 1 audio filter
populated 1 video codec
populated 0 video filters
baresip ready.
bare@192.168.1.66: {0/udp/v4} 200 ok (asterisk pbx 11.13.1~dfsg-2+b1) [1 binding]
1 useragent registered successfully! (16 ms)
> 5555
call: connecting 'sip:5555@192.168.1.66'..
call: sip progress: 100 trying (/)
call: sip progress: 180 ringing (/)
main: long async blocking: 3064>100 ms (h=0xb6e96674 arg=0xa2d778)
audio: set audio decoder: pcmu 8000hz 1ch
audio: set audio encoder: pcmu 8000hz 1ch
audio tx pipeline: alsa ---> vumeter ---> pcmu
audio rx pipeline: alsa <--- vumeter <--- pcmu
set video encoder: h264 packetization-mode=0 (1000000000 bit/s, 24 fps)
gst_video: video encoder h264: 24 fps, 1000000000 bit/s, pktsize=1024
video: not set vidisp '': no such file or directory
v4l2: /dev/video0: found valid v4l2 device (432 x 240) pixfmt=yuyv
v4l2: current input: "camera 1"
bare@192.168.1.66: call established: sip:5555@192.168.1.66
appsrc name=source is-live=true block=true do-timestamp=true max-bytes=1000000 ! videoparse width=%d height=%d format=i420 framerate=%d/1 ! decodebin ! videoconvert ! omxh264enc ! appsink name=sink emit-signals=true drop=true

!!!!!!

appsrc name=source is-live=true block=true do-timestamp=true max-bytes=1000000 ! videoparse width=432 height=240 format=i420 framerate=24/1 ! decodebin ! videoconvert ! omxh264enc ! appsink name=sink emit-signals=true drop=true

!!!!!!

main: long async blocking: 841>100 ms (h=0xb6e96674 arg=0xa2d778)
call: got re-invite (sdp offer) [ ]
call: terminate call '6ff17a4934b7de18' sip:5555@192.168.1.66 ] [ ]
sip:bare@192.168.1.66: call sip:5555@192.168.1.66 terminated (duration: 10 secs)
main: long async blocking: 301>100 ms (h=0xb6ecaa28 arg=0xa2ef80)

raspberrypi