[Libevent-users] Re: free() error: invalid next size (normal) in evbuffer

On 2013. 9. 2., at ìì 1:13, ììì <taxihighway@xxxxxxxxx> wrote:

Hi,
I'm using evtimer which is invoked per 1sec.
In that timer, I call bufferevent_write to socket which is connected to other servers.
I have 3 servers that communicated with each other.

The timer works well in normal state.
But, after I send some data to other servers by bufferevent_write and other servers receives that data by evbuffer_copyout and evbuffer_drain.. (not in timer.)

the timer doesn't work well.

After the timer is invoked.. bufferevent_write is called.. the timer function is returned..
the program dies.. with below errors

*** glibc detected *** ./nstore_cs: free(): invalid next size (normal): 0x00007f332c002630 ***
======= Backtrace: =========
/lib64/libc.so.6[0x3152e75916]
/lib64/libc.so.6[0x3152e78443]
./nstore_cs[0x4ae2f5]
./nstore_cs[0x4111df]
./nstore_cs[0x4b08a2]
./nstore_cs[0x4a8ec1]
./nstore_cs[0x4130b2]
/lib64/libpthread.so.0[0x3153207851]
/lib64/libc.so.6(clone+0x6d)[0x3152ee811d]
======= Memory map: ========
00400000-00587000 r-xp 00000000 08:03 529208 /home1/youngjoon/nbase-m-2.0/sbin/nstore_cs
00787000-0078b000 rw-p 00187000 08:03 529208 /home1/youngjoon/nbase-m-2.0/sbin/nstore_cs
0078b000-00cea000 rw-p 00000000 00:00 0
015a6000-01d16000 rw-p 00000000 00:00 0 [heap]
3152600000-3152620000 r-xp 00000000 08:01 3919 /lib64/ld-2.12.so
315281f000-3152820000 r--p 0001f000 08:01 3919 /lib64/ld-2.12.so
3152820000-3152821000 rw-p 00020000 08:01 3919 /lib64/ld-2.12.so
3152821000-3152822000 rw-p 00000000 00:00 0
3152a00000-3152a02000 r-xp 00000000 08:01 6613 /lib64/libdl-2.12.so
3152a02000-3152c02000 ---p 00002000 08:01 6613 /lib64/libdl-2.12.so
3152c02000-3152c03000 r--p 00002000 08:01 6613 /lib64/libdl-2.12.so
3152c03000-3152c04000 rw-p 00003000 08:01 6613 /lib64/libdl-2.12.so
3152e00000-3152f89000 r-xp 00000000 08:01 6598 /lib64/libc-2.12.so
3152f89000-3153189000 ---p 00189000 08:01 6598 /lib64/libc-2.12.so
3153189000-315318d000 r--p 00189000 08:01 6598 /lib64/libc-2.12.so
315318d000-315318e000 rw-p 0018d000 08:01 6598 /lib64/libc-2.12.so
315318e000-3153193000 rw-p 00000000 00:00 0
3153200000-3153217000 r-xp 00000000 08:01 6600 /lib64/libpthread-2.12.so
3153217000-3153417000 ---p 00017000 08:01 6600 /lib64/libpthread-2.12.so
3153417000-3153418000 r--p 00017000 08:01 6600 /lib64/libpthread-2.12.so
3153418000-3153419000 rw-p 00018000 08:01 6600 /lib64/libpthread-2.12.so
3153419000-315341d000 rw-p 00000000 00:00 0
3153600000-3153615000 r-xp 00000000 08:01 6618 /lib64/libz.so.1.2.3
3153615000-3153814000 ---p 00015000 08:01 6618 /lib64/libz.so.1.2.3
3153814000-3153815000 r--p 00014000 08:01 6618 /lib64/libz.so.1.2.3
3153815000-3153816000 rw-p 00015000 08:01 6618 /lib64/libz.so.1.2.3
3153a00000-3153a83000 r-xp 00000000 08:01 425 /lib64/libm-2.12.so
3153a83000-3153c82000 ---p 00083000 08:01 425 /lib64/libm-2.12.so
3153c82000-3153c83000 r--p 00082000 08:01 425 /lib64/libm-2.12.so
3153c83000-3153c84000 rw-p 00083000 08:01 425 /lib64/libm-2.12.so
3153e00000-3153ee8000 r-xp 00000000 08:01 142990 /usr/lib64/libstdc++.so.6.0.13
3153ee8000-31540e8000 ---p 000e8000 08:01 142990 /usr/lib64/libstdc++.so.6.0.13
31540e8000-31540ef000 r--p 000e8000 08:01 142990 /usr/lib64/libstdc++.so.6.0.13
31540ef000-31540f1000 rw-p 000ef000 08:01 142990 /usr/lib64/libstdc++.so.6.0.13
31540f1000-3154106000 rw-p 00000000 00:00 0
3154e00000-3154e22000 r-xp 00000000 08:01 123 /lib64/libncurses.so.5.7
3154e22000-3155021000 ---p 00022000 08:01 123 /lib64/libncurses.so.5.7
3155021000-3155022000 rw-p 00021000 08:01 123 /lib64/libncurses.so.5.7
3155200000-315521d000 r-xp 00000000 08:01 389 /lib64/libtinfo.so.5.7
315521d000-315541d000 ---p 0001d000 08:01 389 /lib64/libtinfo.so.5.7
315541d000-3155421000 rw-p 0001d000 08:01 389 /lib64/libtinfo.so.5.7
3159e00000-3159e03000 r-xp 00000000 08:01 127 /lib64/libgpg-error.so.0.5.0
3159e03000-315a002000 ---p 00003000 08:01 127 /lib64/libgpg-error.so.0.5.0
315a002000-315a003000 r--p 00002000 08:01 127 /lib64/libgpg-error.so.0.5.0
315a003000-315a004000 rw-p 00003000 08:01 127 /lib64/libgpg-error.so.0.5.0
315a200000-315a272000 r-xp 00000000 08:01 143 /lib64/libgcrypt.so.11.5.3
315a272000-315a471000 ---p 00072000 08:01 143 /lib64/libgcrypt.so.11.5.3
315a471000-315a472000 r--p 00071000 08:01 143 /lib64/libgcrypt.so.11.5.3
315a472000-315a475000 rw-p 00072000 08:01 143 /lib64/libgcrypt.so.11.5.3
7f32f8000000-7f32f8021000 rw-p 00000000 00:00 0
7f32f8021000-7f32fc000000 ---p 00000000 00:00 0
7f32ff9b9000-7f32ff9ba000 ---p 00000000 00:00 0
7f32ff9ba000-7f33003ba000 rw-p 00000000 00:00 0
7f33003ba000-7f33003bb000 ---p 00000000 00:00 0
7f33003bb000-7f3300dbb000 rw-p 00000000 00:00 0
7f3300dbb000-7f3300dbc000 ---p 00000000 00:00 0
7f3300dbc000-7f33017bc000 rw-p 00000000 00:00 0
7f33017bc000-7f33017bd000 ---p 00000000 00:00 0
7f33017bd000-7f33021bd000 rw-p 00000000 00:00 0
7f33021bd000-7f33021be000 ---p 00000000 00:00 0
7f33021be000-7f3302bbe000 rw-p 00000000 00:00 0
7f3302bbe000-7f3302bbf000 ---p 00000000 00:00 0
7f3302bbf000-7f33035bf000 rw-p 00000000 00:00 0
7f33035bf000-7f33035c0000 ---p 00000000 00:00 0
7f33035c0000-7f3303fc0000 rw-p 00000000 00:00 0
7f3303fc0000-7f3303fc1000 ---p 00000000 00:00 0
7f3303fc1000-7f33049c1000 rw-p 00000000 00:00 0
7f33049c1000-7f33049c2000 ---p 00000000 00:00 0
7f33049c2000-7f33053c2000 rw-p 00000000 00:00 0
7f33053c2000-7f33053c3000 ---p 00000000 00:00 0
7f33053c3000-7f3305dc3000 rw-p 00000000 00:00 0
7f3305dc3000-7f3305dc4000 ---p 00000000 00:00 0
7f3305dc4000-7f33067c4000 rw-p 00000000 00:00 0
7f33067c4000-7f33067c5000 ---p 00000000 00:00 0
7f33067c5000-7f33071c5000 rw-p 00000000 00:00 0
7f33071c5000-7f33071c6000 ---p 00000000 00:00 0
7f33071c6000-7f3307bc6000 rw-p 00000000 00:00 0
7f3307bc6000-7f3307bc7000 ---p 00000000 00:00 0
7f3307bc7000-7f33085c7000 rw-p 00000000 00:00 0
7f33085c7000-7f33085c8000 ---p 00000000 00:00 0
7f33085c8000-7f3308fc8000 rw-p 00000000 00:00 0
7f3308fc8000-7f3308fc9000 ---p 00000000 00:00 0
7f3308fc9000-7f33099c9000 rw-p 00000000 00:00 0
7f33099c9000-7f33099ca000 ---p 00000000 00:00 0
7f33099ca000-7f330a3ca000 rw-p 00000000 00:00 0
7f330a3ca000-7f330a3cb000 ---p 00000000 00:00 0
7f330a3cb000-7f330adcb000 rw-p 00000000 00:00 0
7f330adcb000-7f330adcc000 ---p 00000000 00:00 0

below is core..

#0 0x0000003152e328a5 in raise () from /lib64/libc.so.6
No symbol table info available.
#1 0x0000003152e34085 in abort () from /lib64/libc.so.6
No symbol table info available.
#2 0x0000003152e6ffe7 in __libc_message () from /lib64/libc.so.6
No symbol table info available.
#3 0x0000003152e75916 in malloc_printerr () from /lib64/libc.so.6
No symbol table info available.
#4 0x0000003152e78443 in _int_free () from /lib64/libc.so.6
No symbol table info available.
#5 0x00000000004ae2f5 in evbuffer_chain_free (buf=0x7f332c000b20, len=<value optimized out>) at buffer.c:226
__func__ = "evbuffer_chain_free"
#6 evbuffer_drain (buf=0x7f332c000b20, len=<value optimized out>) at buffer.c:924
chain = 0x7f332c002630
next = 0x0
remaining = <value optimized out>
old_len = 32
result = <value optimized out>
__func__ = "evbuffer_drain"
#7 0x00000000004111df in _ns_rep_tcp_read (bev=0x7f332c000920, arg=0xa9b4c0) at ns_replicator.c:627
size_with_body = 32
ret = 0
rep = 0xa9b4c0
msg = {cmd = NS_REP_TCP_COMMIT_DUMMY, cgid = 99, cg_ver = 1, cgitem_follow = 0, seq = 0, body_size = 0}
res = 32
i = 960
is_rep = {0 <repeats 100 times>}
input_buf = 0x7f332c000b20
input_buf_len = 32
__FUNCTION__ = "_ns_rep_tcp_read"
#8 0x00000000004b08a2 in bufferevent_readcb (fd=37, event=<value optimized out>, arg=0x7f332c000920) at bufferevent_sock.c:182
bufev = 0x3329
bufev_p = <value optimized out>
input = 0x7f332c000b20
res = 960
what = <value optimized out>
howmuch = <value optimized out>
readmax = <value optimized out>
#9 0x00000000004a8ec1 in event_process_active_single_queue (base=0x1cec730, flags=0) at event.c:1308
No locals.
#10 event_process_active (base=0x1cec730, flags=0) at event.c:1375
i = 0
#11 event_base_loop (base=0x1cec730, flags=0) at event.c:1572
n = 1
evsel = 0x553dc0
tv = {tv_sec = 5, tv_usec = 0}
tv_p = 0x0
res = <value optimized out>
retval = <value optimized out>
__func__ = "event_base_loop"
#12 0x00000000004130b2 in ns_replicator (arg=0xce6ff0) at ns_replicator.c:1191
rep = 0xa9b4c0
pThr = 0xce6ff0
__PRETTY_FUNCTION__ = "ns_replicator"
__FUNCTION__ = "ns_replicator"
#13 0x0000003153207851 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#14 0x0000003152ee811d in clone () from /lib64/libc.so.6
No symbol table info available.

I think the program dies when the data in output buffer are sent to TCP socket.
Also, other server that receives data in timer dies because of below errors.

*** glibc detected *** ./nstore_cs: free(): invalid next size (normal): 0x00007f2bac000e10 ***
======= Backtrace: =========
/lib64/libc.so.6[0x39f4c75916]
/lib64/libc.so.6[0x39f4c78443]
./nstore_cs[0x4ae2f5]
./nstore_cs[0x4aeb2c]
./nstore_cs[0x4b06a5]
./nstore_cs[0x4a8ec1]
./nstore_cs[0x4130b2]
/lib64/libpthread.so.0[0x39f5007851]
/lib64/libc.so.6(clone+0x6d)[0x39f4ce811d]
======= Memory map: ========
00400000-00587000 r-xp 00000000 08:03 529210 /home1/youngjoon/nbase-m-2.0/sbin/nstore_cs
00787000-0078b000 rw-p 00187000 08:03 529210 /home1/youngjoon/nbase-m-2.0/sbin/nstore_cs
0078b000-00cea000 rw-p 00000000 00:00 0
0102f000-0179f000 rw-p 00000000 00:00 0 [heap]
39f4400000-39f4420000 r-xp 00000000 08:01 10130 /lib64/ld-2.12.so
39f461f000-39f4620000 r--p 0001f000 08:01 10130 /lib64/ld-2.12.so
39f4620000-39f4621000 rw-p 00020000 08:01 10130 /lib64/ld-2.12.so
39f4621000-39f4622000 rw-p 00000000 00:00 0
39f4800000-39f4802000 r-xp 00000000 08:01 10133 /lib64/libdl-2.12.so
39f4802000-39f4a02000 ---p 00002000 08:01 10133 /lib64/libdl-2.12.so
39f4a02000-39f4a03000 r--p 00002000 08:01 10133 /lib64/libdl-2.12.so
39f4a03000-39f4a04000 rw-p 00003000 08:01 10133 /lib64/libdl-2.12.so
39f4c00000-39f4d89000 r-xp 00000000 08:01 10131 /lib64/libc-2.12.so
39f4d89000-39f4f89000 ---p 00189000 08:01 10131 /lib64/libc-2.12.so
39f4f89000-39f4f8d000 r--p 00189000 08:01 10131 /lib64/libc-2.12.so
39f4f8d000-39f4f8e000 rw-p 0018d000 08:01 10131 /lib64/libc-2.12.so
39f4f8e000-39f4f93000 rw-p 00000000 00:00 0
39f5000000-39f5017000 r-xp 00000000 08:01 1742 /lib64/libpthread-2.12.so
39f5017000-39f5217000 ---p 00017000 08:01 1742 /lib64/libpthread-2.12.so
39f5217000-39f5218000 r--p 00017000 08:01 1742 /lib64/libpthread-2.12.so
39f5218000-39f5219000 rw-p 00018000 08:01 1742 /lib64/libpthread-2.12.so
39f5219000-39f521d000 rw-p 00000000 00:00 0
39f5400000-39f5415000 r-xp 00000000 08:01 3622 /lib64/libz.so.1.2.3
39f5415000-39f5614000 ---p 00015000 08:01 3622 /lib64/libz.so.1.2.3
39f5614000-39f5615000 r--p 00014000 08:01 3622 /lib64/libz.so.1.2.3
39f5615000-39f5616000 rw-p 00015000 08:01 3622 /lib64/libz.so.1.2.3
39f5800000-39f5883000 r-xp 00000000 08:01 10147 /lib64/libm-2.12.so
39f5883000-39f5a82000 ---p 00083000 08:01 10147 /lib64/libm-2.12.so
39f5a82000-39f5a83000 r--p 00082000 08:01 10147 /lib64/libm-2.12.so
39f5a83000-39f5a84000 rw-p 00083000 08:01 10147 /lib64/libm-2.12.so
39f5c00000-39f5ce8000 r-xp 00000000 08:01 536101 /usr/lib64/libstdc++.so.6.0.13
39f5ce8000-39f5ee8000 ---p 000e8000 08:01 536101 /usr/lib64/libstdc++.so.6.0.13
39f5ee8000-39f5eef000 r--p 000e8000 08:01 536101 /usr/lib64/libstdc++.so.6.0.13
39f5eef000-39f5ef1000 rw-p 000ef000 08:01 536101 /usr/lib64/libstdc++.so.6.0.13
39f5ef1000-39f5f06000 rw-p 00000000 00:00 0
39f6c00000-39f6c22000 r-xp 00000000 08:01 1724 /lib64/libncurses.so.5.7
39f6c22000-39f6e21000 ---p 00022000 08:01 1724 /lib64/libncurses.so.5.7
39f6e21000-39f6e22000 rw-p 00021000 08:01 1724 /lib64/libncurses.so.5.7
39f7000000-39f701d000 r-xp 00000000 08:01 1601 /lib64/libtinfo.so.5.7
39f701d000-39f721d000 ---p 0001d000 08:01 1601 /lib64/libtinfo.so.5.7
39f721d000-39f7221000 rw-p 0001d000 08:01 1601 /lib64/libtinfo.so.5.7
39fbc00000-39fbc03000 r-xp 00000000 08:01 1736 /lib64/libgpg-error.so.0.5.0
39fbc03000-39fbe02000 ---p 00003000 08:01 1736 /lib64/libgpg-error.so.0.5.0
39fbe02000-39fbe03000 r--p 00002000 08:01 1736 /lib64/libgpg-error.so.0.5.0
39fbe03000-39fbe04000 rw-p 00003000 08:01 1736 /lib64/libgpg-error.so.0.5.0
39fc000000-39fc072000 r-xp 00000000 08:01 2252 /lib64/libgcrypt.so.11.5.3
39fc072000-39fc271000 ---p 00072000 08:01 2252 /lib64/libgcrypt.so.11.5.3
39fc271000-39fc272000 r--p 00071000 08:01 2252 /lib64/libgcrypt.so.11.5.3
39fc272000-39fc275000 rw-p 00072000 08:01 2252 /lib64/libgcrypt.so.11.5.3
7f2b64000000-7f2b64021000 rw-p 00000000 00:00 0
7f2b64021000-7f2b68000000 ---p 00000000 00:00 0
7f2b6b59f000-7f2b6b5a0000 ---p 00000000 00:00 0
7f2b6b5a0000-7f2b6bfa0000 rw-p 00000000 00:00 0
7f2b6bfa0000-7f2b6bfa1000 ---p 00000000 00:00 0
7f2b6bfa1000-7f2b6c9a1000 rw-p 00000000 00:00 0
7f2b6c9a1000-7f2b6c9a2000 ---p 00000000 00:00 0
7f2b6c9a2000-7f2b6d3a2000 rw-p 00000000 00:00 0
7f2b6d3a2000-7f2b6d3a3000 ---p 00000000 00:00 0
7f2b6d3a3000-7f2b6dda3000 rw-p 00000000 00:00 0
7f2b6dda3000-7f2b6dda4000 ---p 00000000 00:00 0
7f2b6dda4000-7f2b6e7a4000 rw-p 00000000 00:00 0
7f2b6e7a4000-7f2b6e7a5000 ---p 00000000 00:00 0
7f2b6e7a5000-7f2b6f1a5000 rw-p 00000000 00:00 0
7f2b6f1a5000-7f2b6f1a6000 ---p 00000000 00:00 0
7f2b6f1a6000-7f2b6fba6000 rw-p 00000000 00:00 0
7f2b6fba6000-7f2b6fba7000 ---p 00000000 00:00 0
7f2b6fba7000-7f2b705a7000 rw-p 00000000 00:00 0
7f2b705a7000-7f2b705a8000 ---p 00000000 00:00 0
7f2b705a8000-7f2b70fa8000 rw-p 00000000 00:00 0
7f2b70fa8000-7f2b70fa9000 ---p 00000000 00:00 0
7f2b70fa9000-7f2b719a9000 rw-p 00000000 00:00 0
7f2b719a9000-7f2b719aa000 ---p 00000000 00:00 0
7f2b719aa000-7f2b723aa000 rw-p 00000000 00:00 0
7f2b723aa000-7f2b723ab000 ---p 00000000 00:00 0
7f2b723ab000-7f2b72dab000 rw-p 00000000 00:00 0
7f2b72dab000-7f2b72dac000 ---p 00000000 00:00 0
7f2b72dac000-7f2b737ac000 rw-p 00000000 00:00 0
7f2b737ac000-7f2b737ad000 ---p 00000000 00:00 0
7f2b737ad000-7f2b741ad000 rw-p 00000000 00:00 0
7f2b741ad000-7f2b741ae000 ---p 00000000 00:00 0
7f2b741ae000-7f2b74bae000 rw-p 00000000 00:00 0
7f2b74bae000-7f2b74baf000 ---p 00000000 00:00 0
7f2b74baf000-7f2b755af000 rw-p 00000000 00:00 0
7f2b755af000-7f2b755b0000 ---p 00000000 00:00 0
7f2b755b0000-7f2b75fb0000 rw-p 00000000 00:00 0
7f2b75fb0000-7f2b75fb1000 ---p 00000000 00:00 0
7f2b75fb1000-7f2b769b1000 rw-p 00000000 00:00 0
7f2b769b1000-7f2b769b2000 ---p 00000000 00:00 0

(gdb) bt full
#0 0x00000039f4c328a5 in raise () from /lib64/libc.so.6
No symbol table info available.
#1 0x00000039f4c34085 in abort () from /lib64/libc.so.6
No symbol table info available.
#2 0x00000039f4c6ffe7 in __libc_message () from /lib64/libc.so.6
No symbol table info available.
#3 0x00000039f4c75916 in malloc_printerr () from /lib64/libc.so.6
No symbol table info available.
#4 0x00000039f4c78443 in _int_free () from /lib64/libc.so.6
No symbol table info available.
#5 0x00000000004ae2f5 in evbuffer_chain_free (buf=0x1782b70, len=<value optimized out>) at buffer.c:226
__func__ = "evbuffer_chain_free"
#6 evbuffer_drain (buf=0x1782b70, len=<value optimized out>) at buffer.c:924
chain = 0x7f2bac000e10
next = 0x0
remaining = <value optimized out>
old_len = 960
result = <value optimized out>
__func__ = "evbuffer_drain"
#7 0x00000000004aeb2c in evbuffer_write_atmost (buffer=0x1782b70, fd=57, howmuch=960) at buffer.c:2314
n = 960
#8 0x00000000004b06a5 in bufferevent_writecb (fd=57, event=<value optimized out>, arg=0x17828d0) at bufferevent_sock.c:260
bufev = <value optimized out>
bufev_p = <value optimized out>
res = <value optimized out>
what = <value optimized out>
connected = 0
atmost = 16384
#9 0x00000000004a8ec1 in event_process_active_single_queue (base=0x1775730, flags=0) at event.c:1308
No locals.
#10 event_process_active (base=0x1775730, flags=0) at event.c:1375
i = 0
#11 event_base_loop (base=0x1775730, flags=0) at event.c:1572
n = 2
evsel = 0x553dc0
tv = {tv_sec = 5, tv_usec = 0}
tv_p = 0x0
res = <value optimized out>
retval = <value optimized out>
__func__ = "event_base_loop"
#12 0x00000000004130b2 in ns_replicator (arg=0xce6ff0) at ns_replicator.c:1191
rep = 0xa9b4c0
pThr = 0xce6ff0
__PRETTY_FUNCTION__ = "ns_replicator"
__FUNCTION__ = "ns_replicator"
#13 0x00000039f5007851 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#14 0x00000039f4ce811d in clone () from /lib64/libc.so.6
No symbol table info available.

I wonder why timer doesn't work well suddenly after I sent and received data.

I used libevent-2.0.21-stable version.
Below is my code.

/********* setup libevent *****************/

// init timer
rep->timer = evtimer_new(rep->evbase, _ns_rep_timer, rep);
evtimer_add(rep->timer, &rep->commit_interval);

// setup bufferevent
struct bufferevent *bev = bufferevent_socket_new(rep->evbase, sock, BEV_OPT_CLOSE_ON_FREE);
bufferevent_setwatermark(bev, EV_READ, sizeof(ns_rep_tcp_msg_header_t), 0);

bufferevent_setwatermark(bev, EV_WRITE, 0, 0);
bufferevent_setcb(bev, _ns_rep_tcp_read, _ns_rep_tcp_write, NULL, rep);
bufferevent_enable(bev, EV_READ|EV_WRITE);

/*************************************************/

timer is used to send a commit-message in commit_interval.
and bufferevent is used to read/write some data or read a commit_message.

/ ********** read callback ********/
static void _ns_rep_tcp_write(struct bufferevent *bev, void *arg)
{
................
/* check the size of input buffer */

/* just get a pointer of input buffer, not remove data from input buffer */

input_buf = bufferevent_get_input(bev);
input_buf_len = evbuffer_get_length(input_buf);
if (input_buf_len < sizeof(ns_rep_tcp_msg_header_t)) {
NO_TRACE("input_buf_len < sizeof(ns_rep_tcp_msg_header_t)(%d)", sizeof(ns_rep_tcp_msg_header_t));

bufferevent_setwatermark(bev, EV_READ, sizeof(ns_rep_tcp_msg_header_t), 0);
break;
}

/* just copyout data from input buffer to check body_size, not remove data from input buffer */

NO_TRACE("evbuffer_copyout. input_buf_len:%d", input_buf_len);
res = evbuffer_copyout(input_buf, &msg, sizeof(ns_rep_tcp_msg_header_t));
if (res != sizeof(ns_rep_tcp_msg_header_t)) {

NO_ERROR(NE_FATAL, "fail to evbuffer_copyout. res:%d", res);
break;
}

/* check the header and body are all in input buffer */

size_t size_with_body = sizeof(ns_rep_tcp_msg_header_t) + msg.body_size;
if (input_buf_len < size_with_body) {
NO_TRACE("input_buf_len < size_with_body(%d)", size_with_body);

NO_TRACE("change read-low-watermark to size_with_body(%d", size_with_body);

/* change watermark to read header and body in next invoked time */
bufferevent_setwatermark(bev, EV_READ, size_with_body, 0);

break;
}

// drain a header in input buffer because a header was copied out before.
NO_TRACE("evbuffer_drain. cgid:%d", msg.cgid);

res = evbuffer_drain(input_buf, sizeof(ns_rep_tcp_msg_header_t));

/************************************************************************************/

/*************** write callback ***********/
static void _ns_rep_tcp_write(struct bufferevent *bev, void *arg)
{
// not used
}
/*********************************************/

/*************** timer *****************/
// timer for send COMMIT
static void _ns_rep_timer(evutil_socket_t fd, short events, void *arg)

{
int i;
for (i = 0 ; i < 2 ; i++) {
bufferevent_write(bev[i], msg, sizeof(ns_rep_tcp_msg_header_t));
}

evtimer_add(rep->timer, &rep->commit_interval);
}
/*****************************************/

timer interval is 1 second.
Why a timer doesn't work well after some data is sent/received by bufferevent read/write callback?

please give me some advise.
thank you.

[Libevent-users] Re: free() error: invalid next size (normal) in evbuffer_drain