I am running into a deadlock with both libevent 2.0 and 2.1 that I cannot explain. One thread is executing a write callback and the other thread a read callback of a different bufferevent. Since they are both accessing the same data structure passed to the callbacks (arg=0x630c40), I am using a rwlock in the data structure. I did not expect though the evbuffer internal lock to interfere with the lock used to synchronize access to the data structure shared between the callbacks.
I am including the backtraces and the relevant portions of code. I would appreciate if someone can give me any suggestion on how to work around this.
Thanks,
-Marco G.
(gdb) t 1
[Switching to thread 1 (Thread 0x7ffff7fdd780 (LWP 12008))]
(gdb) bt
#0 Â__lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 Â0x00007ffff7770672 in _L_lock_953 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 Â0x00007ffff77704da in __GI___pthread_mutex_lock (mutex=0x620200) at ../nptl/pthread_mutex_lock.c:114
#3 Â0x00007ffff7b9686e in evbuffer_get_length (buffer=0x630930) at buffer.c:610
#4 Â0x000000000040494b in device_write_cb (bev=0x123f640, arg=0x630c40) at cometa.c:667
#5 Â0x00007ffff7ba2139 in bufferevent_trigger_nolock_ (options=0, iotype=4, bufev=0x123f640) at bufferevent-internal.h:369
#6 Âbufferevent_writecb (fd=<optimized out>, event=<optimized out>, arg=0x123f640) at bufferevent_sock.c:297
#7 Â0x00007ffff7ba7a19 in event_persist_closure (ev=<optimized out>, base=0x6218d0) at event.c:1531
#8 Âevent_process_active_single_queue (base=base@entry=0x6218d0, activeq=0x621d20, max_to_process=max_to_process@entry=2147483647,Â
  endtime=endtime@entry=0x0) at event.c:1590
#9 Â0x00007ffff7ba82cf in event_process_active (base=0x6218d0) at event.c:1689
#10 event_base_loop (base=0x6218d0, flags=0) at event.c:1912
#11 0x000000000040a700 in main () at cometa.c:1571
bufferevent 0x123f640 write callback:
665 Â Âpthread_rwlock_wrlock(&device->lock);
666 Â Âif ((device->pending && device->state == WAITING_STATE) || device->ws_pending) {
667 Â Â int l = evbuffer_get_length(device_bev); ÂÂ
668 Â Â pthread_rwlock_unlock(&device->lock);
669 Â Â output = bufferevent_get_output(device->pending->bufev);
670 if (evbuffer_get_length(output) == 0) {
671 /* free and close the connection */
672 pthread_rwlock_wrlock(&device->lock);
673 Â Â free_http_connection(device->pending);
674 Â Â device->pending = NULL;
675 Â Â device->state = IDLE_STATE;
676 Â device->chunk_length = 0;
677 Â Â device->ntoread = -1;
678
679 Â Â evbuffer_drain (device_bev, l);
680 Â sem_post(&device->active);
681 Â pthread_rwlock_unlock(&device->lock);
682 Â Â }
683 Â Â} else {
684 Â Â pthread_rwlock_unlock(&device->lock);
685 Â Â}
(gdb) t 3
[Switching to thread 3 (Thread 0x7ffff532f700 (LWP 12013))]
(gdb) bt
#0 Âpthread_rwlock_wrlock () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S:85
#1 Â0x0000000000404667 in device_read_cb (bev=0x630730, arg=0x630c40) at cometa.c:604
#2 Â0x00007ffff7ba2324 in bufferevent_trigger_nolock_ (options=0, iotype=2, bufev=0x630730) at bufferevent-internal.h:366
#3 Âbufferevent_readcb (fd=34, event=<optimized out>, arg=0x630730) at bufferevent_sock.c:187
#4 Â0x00007ffff7ba7a19 in event_persist_closure (ev=<optimized out>, base=0x7fffe8000900) at event.c:1531
#5 Âevent_process_active_single_queue (base=base@entry=0x7fffe8000900, activeq=0x7fffe8000d50,Â
  max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0) at event.c:1590
#6 Â0x00007ffff7ba82cf in event_process_active (base=0x7fffe8000900) at event.c:1689
#7 Âevent_base_loop (base=0x7fffe8000900, flags=0) at event.c:1912
#8 Â0x000000000040a1ff in device_loop (arg=0x0) at cometa.c:1437
#9 Â0x00007ffff776e182 in start_thread (arg=0x7ffff532f700) at pthread_create.c:312
#10 0x00007ffff689700d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
bufferevent 0x630730 read call back:
604 pthread_rwlock_wrlock(&device->lock);Â
605 Â Âif (device->state != WAITING_STATE) {
606 Â Â
607 Â device->chunk_length = 0;
608 Â Â device->ntoread = -1;
609 Â Â /* in case of state == WAITING_STATE the counters reset is done in the write callback after sending the response */
610 Â Â}
611 pthread_rwlock_unlock(&device->lock);