[Libevent-users] free() error: invalid next size (normal) in evbuffer_drain

[이영준 <taxihighway@xxxxxxxxx>]

Hi, 

I'm using evtimer which is invoked per 1sec.

In that timer, I call bufferevent_write to socket which is connected to other servers. 

I have 3 servers that communicated with each other.

The timer works well in normal state. 

But, after I send some data to other servers by bufferevent_write and other servers receives that data by evbuffer_copyout and evbuffer_drain.. (not in timer.)

the timer doesn't work well.

After the timer is invoked.. bufferevent_write is called.. the timer function is returned.. 

the program dies.. with below errors

*** glibc detected *** ./nstore_cs: free(): invalid next size (normal): 0x00007f332c002630 ***

======= Backtrace: =========

/lib64/libc.so.6[0x3152e75916]

/lib64/libc.so.6[0x3152e78443]

./nstore_cs[0x4ae2f5]

./nstore_cs[0x4111df]

./nstore_cs[0x4b08a2]

./nstore_cs[0x4a8ec1]

./nstore_cs[0x4130b2]

/lib64/libpthread.so.0[0x3153207851]

/lib64/libc.so.6(clone+0x6d)[0x3152ee811d]

======= Memory map: ========

00400000-00587000 r-xp 00000000 08:03 529208                             /home1/youngjoon/nbase-m-2.0/sbin/nstore_cs

00787000-0078b000 rw-p 00187000 08:03 529208                             /home1/youngjoon/nbase-m-2.0/sbin/nstore_cs

0078b000-00cea000 rw-p 00000000 00:00 0

015a6000-01d16000 rw-p 00000000 00:00 0                                  [heap]

3152600000-3152620000 r-xp 00000000 08:01 3919                           /lib64/ld-2.12.so

315281f000-3152820000 r--p 0001f000 08:01 3919                           /lib64/ld-2.12.so

3152820000-3152821000 rw-p 00020000 08:01 3919                           /lib64/ld-2.12.so

3152821000-3152822000 rw-p 00000000 00:00 0

3152a00000-3152a02000 r-xp 00000000 08:01 6613                           /lib64/libdl-2.12.so

3152a02000-3152c02000 ---p 00002000 08:01 6613                           /lib64/libdl-2.12.so

3152c02000-3152c03000 r--p 00002000 08:01 6613                           /lib64/libdl-2.12.so

3152c03000-3152c04000 rw-p 00003000 08:01 6613                           /lib64/libdl-2.12.so

3152e00000-3152f89000 r-xp 00000000 08:01 6598                           /lib64/libc-2.12.so

3152f89000-3153189000 ---p 00189000 08:01 6598                           /lib64/libc-2.12.so

3153189000-315318d000 r--p 00189000 08:01 6598                           /lib64/libc-2.12.so

315318d000-315318e000 rw-p 0018d000 08:01 6598                           /lib64/libc-2.12.so

315318e000-3153193000 rw-p 00000000 00:00 0

3153200000-3153217000 r-xp 00000000 08:01 6600                           /lib64/libpthread-2.12.so

3153217000-3153417000 ---p 00017000 08:01 6600                           /lib64/libpthread-2.12.so

3153417000-3153418000 r--p 00017000 08:01 6600                           /lib64/libpthread-2.12.so

3153418000-3153419000 rw-p 00018000 08:01 6600                           /lib64/libpthread-2.12.so

3153419000-315341d000 rw-p 00000000 00:00 0

3153600000-3153615000 r-xp 00000000 08:01 6618                           /lib64/libz.so.1.2.3

3153615000-3153814000 ---p 00015000 08:01 6618                           /lib64/libz.so.1.2.3

3153814000-3153815000 r--p 00014000 08:01 6618                           /lib64/libz.so.1.2.3

3153815000-3153816000 rw-p 00015000 08:01 6618                           /lib64/libz.so.1.2.3

3153a00000-3153a83000 r-xp 00000000 08:01 425                            /lib64/libm-2.12.so

3153a83000-3153c82000 ---p 00083000 08:01 425                            /lib64/libm-2.12.so

3153c82000-3153c83000 r--p 00082000 08:01 425                            /lib64/libm-2.12.so

3153c83000-3153c84000 rw-p 00083000 08:01 425                            /lib64/libm-2.12.so

3153e00000-3153ee8000 r-xp 00000000 08:01 142990                         /usr/lib64/libstdc++.so.6.0.13

3153ee8000-31540e8000 ---p 000e8000 08:01 142990                         /usr/lib64/libstdc++.so.6.0.13

31540e8000-31540ef000 r--p 000e8000 08:01 142990                         /usr/lib64/libstdc++.so.6.0.13

31540ef000-31540f1000 rw-p 000ef000 08:01 142990                         /usr/lib64/libstdc++.so.6.0.13

31540f1000-3154106000 rw-p 00000000 00:00 0

3154e00000-3154e22000 r-xp 00000000 08:01 123                            /lib64/libncurses.so.5.7

3154e22000-3155021000 ---p 00022000 08:01 123                            /lib64/libncurses.so.5.7

3155021000-3155022000 rw-p 00021000 08:01 123                            /lib64/libncurses.so.5.7

3155200000-315521d000 r-xp 00000000 08:01 389                            /lib64/libtinfo.so.5.7

315521d000-315541d000 ---p 0001d000 08:01 389                            /lib64/libtinfo.so.5.7

315541d000-3155421000 rw-p 0001d000 08:01 389                            /lib64/libtinfo.so.5.7

3159e00000-3159e03000 r-xp 00000000 08:01 127                            /lib64/libgpg-error.so.0.5.0

3159e03000-315a002000 ---p 00003000 08:01 127                            /lib64/libgpg-error.so.0.5.0

315a002000-315a003000 r--p 00002000 08:01 127                            /lib64/libgpg-error.so.0.5.0

315a003000-315a004000 rw-p 00003000 08:01 127                            /lib64/libgpg-error.so.0.5.0

315a200000-315a272000 r-xp 00000000 08:01 143                            /lib64/libgcrypt.so.11.5.3

315a272000-315a471000 ---p 00072000 08:01 143                            /lib64/libgcrypt.so.11.5.3

315a471000-315a472000 r--p 00071000 08:01 143                            /lib64/libgcrypt.so.11.5.3

315a472000-315a475000 rw-p 00072000 08:01 143                            /lib64/libgcrypt.so.11.5.3

7f32f8000000-7f32f8021000 rw-p 00000000 00:00 0

7f32f8021000-7f32fc000000 ---p 00000000 00:00 0

7f32ff9b9000-7f32ff9ba000 ---p 00000000 00:00 0

7f32ff9ba000-7f33003ba000 rw-p 00000000 00:00 0

7f33003ba000-7f33003bb000 ---p 00000000 00:00 0

7f33003bb000-7f3300dbb000 rw-p 00000000 00:00 0

7f3300dbb000-7f3300dbc000 ---p 00000000 00:00 0

7f3300dbc000-7f33017bc000 rw-p 00000000 00:00 0

7f33017bc000-7f33017bd000 ---p 00000000 00:00 0

7f33017bd000-7f33021bd000 rw-p 00000000 00:00 0

7f33021bd000-7f33021be000 ---p 00000000 00:00 0

7f33021be000-7f3302bbe000 rw-p 00000000 00:00 0

7f3302bbe000-7f3302bbf000 ---p 00000000 00:00 0

7f3302bbf000-7f33035bf000 rw-p 00000000 00:00 0

7f33035bf000-7f33035c0000 ---p 00000000 00:00 0

7f33035c0000-7f3303fc0000 rw-p 00000000 00:00 0

7f3303fc0000-7f3303fc1000 ---p 00000000 00:00 0

7f3303fc1000-7f33049c1000 rw-p 00000000 00:00 0

7f33049c1000-7f33049c2000 ---p 00000000 00:00 0

7f33049c2000-7f33053c2000 rw-p 00000000 00:00 0

7f33053c2000-7f33053c3000 ---p 00000000 00:00 0

7f33053c3000-7f3305dc3000 rw-p 00000000 00:00 0

7f3305dc3000-7f3305dc4000 ---p 00000000 00:00 0

7f3305dc4000-7f33067c4000 rw-p 00000000 00:00 0

7f33067c4000-7f33067c5000 ---p 00000000 00:00 0

7f33067c5000-7f33071c5000 rw-p 00000000 00:00 0

7f33071c5000-7f33071c6000 ---p 00000000 00:00 0

7f33071c6000-7f3307bc6000 rw-p 00000000 00:00 0

7f3307bc6000-7f3307bc7000 ---p 00000000 00:00 0

7f3307bc7000-7f33085c7000 rw-p 00000000 00:00 0

7f33085c7000-7f33085c8000 ---p 00000000 00:00 0

7f33085c8000-7f3308fc8000 rw-p 00000000 00:00 0

7f3308fc8000-7f3308fc9000 ---p 00000000 00:00 0

7f3308fc9000-7f33099c9000 rw-p 00000000 00:00 0

7f33099c9000-7f33099ca000 ---p 00000000 00:00 0

7f33099ca000-7f330a3ca000 rw-p 00000000 00:00 0

7f330a3ca000-7f330a3cb000 ---p 00000000 00:00 0

7f330a3cb000-7f330adcb000 rw-p 00000000 00:00 0

7f330adcb000-7f330adcc000 ---p 00000000 00:00 0

below is core..

#0  0x0000003152e328a5 in raise () from /lib64/libc.so.6

No symbol table info available.

#1  0x0000003152e34085 in abort () from /lib64/libc.so.6

No symbol table info available.

#2  0x0000003152e6ffe7 in __libc_message () from /lib64/libc.so.6

No symbol table info available.

#3  0x0000003152e75916 in malloc_printerr () from /lib64/libc.so.6

No symbol table info available.

#4  0x0000003152e78443 in _int_free () from /lib64/libc.so.6

No symbol table info available.

#5  0x00000000004ae2f5 in evbuffer_chain_free (buf=0x7f332c000b20, len=<value optimized out>) at buffer.c:226

        __func__ = "evbuffer_chain_free"

#6  evbuffer_drain (buf=0x7f332c000b20, len=<value optimized out>) at buffer.c:924

        chain = 0x7f332c002630

        next = 0x0

        remaining = <value optimized out>

        old_len = 32

        result = <value optimized out>

        __func__ = "evbuffer_drain"

#7  0x00000000004111df in _ns_rep_tcp_read (bev=0x7f332c000920, arg=0xa9b4c0) at ns_replicator.c:627

        size_with_body = 32

        ret = 0

        rep = 0xa9b4c0

        msg = {cmd = NS_REP_TCP_COMMIT_DUMMY, cgid = 99, cg_ver = 1, cgitem_follow = 0, seq = 0, body_size = 0}

        res = 32

        i = 960

        is_rep = {0 <repeats 100 times>}

        input_buf = 0x7f332c000b20

        input_buf_len = 32

        __FUNCTION__ = "_ns_rep_tcp_read"

#8  0x00000000004b08a2 in bufferevent_readcb (fd=37, event=<value optimized out>, arg=0x7f332c000920) at bufferevent_sock.c:182

        bufev = 0x3329

        bufev_p = <value optimized out>

        input = 0x7f332c000b20

        res = 960

        what = <value optimized out>

        howmuch = <value optimized out>

        readmax = <value optimized out>

#9  0x00000000004a8ec1 in event_process_active_single_queue (base=0x1cec730, flags=0) at event.c:1308

No locals.

#10 event_process_active (base=0x1cec730, flags=0) at event.c:1375

        i = 0

#11 event_base_loop (base=0x1cec730, flags=0) at event.c:1572

        n = 1

        evsel = 0x553dc0

        tv = {tv_sec = 5, tv_usec = 0}

        tv_p = 0x0

        res = <value optimized out>

        retval = <value optimized out>

        __func__ = "event_base_loop"

#12 0x00000000004130b2 in ns_replicator (arg=0xce6ff0) at ns_replicator.c:1191

        rep = 0xa9b4c0

        pThr = 0xce6ff0

        __PRETTY_FUNCTION__ = "ns_replicator"

        __FUNCTION__ = "ns_replicator"

#13 0x0000003153207851 in start_thread () from /lib64/libpthread.so.0

No symbol table info available.

#14 0x0000003152ee811d in clone () from /lib64/libc.so.6

No symbol table info available.

I think the program dies when the data in output buffer are sent to TCP socket.

Also, other server that receives data in timer dies because of below errors.

*** glibc detected *** ./nstore_cs: free(): invalid next size (normal): 0x00007f2bac000e10 ***

======= Backtrace: =========

/lib64/libc.so.6[0x39f4c75916]

/lib64/libc.so.6[0x39f4c78443]

./nstore_cs[0x4ae2f5]

./nstore_cs[0x4aeb2c]

./nstore_cs[0x4b06a5]

./nstore_cs[0x4a8ec1]

./nstore_cs[0x4130b2]

/lib64/libpthread.so.0[0x39f5007851]

/lib64/libc.so.6(clone+0x6d)[0x39f4ce811d]

======= Memory map: ========

00400000-00587000 r-xp 00000000 08:03 529210                             /home1/youngjoon/nbase-m-2.0/sbin/nstore_cs

00787000-0078b000 rw-p 00187000 08:03 529210                             /home1/youngjoon/nbase-m-2.0/sbin/nstore_cs

0078b000-00cea000 rw-p 00000000 00:00 0

0102f000-0179f000 rw-p 00000000 00:00 0                                  [heap]

39f4400000-39f4420000 r-xp 00000000 08:01 10130                          /lib64/ld-2.12.so

39f461f000-39f4620000 r--p 0001f000 08:01 10130                          /lib64/ld-2.12.so

39f4620000-39f4621000 rw-p 00020000 08:01 10130                          /lib64/ld-2.12.so

39f4621000-39f4622000 rw-p 00000000 00:00 0

39f4800000-39f4802000 r-xp 00000000 08:01 10133                          /lib64/libdl-2.12.so

39f4802000-39f4a02000 ---p 00002000 08:01 10133                          /lib64/libdl-2.12.so

39f4a02000-39f4a03000 r--p 00002000 08:01 10133                          /lib64/libdl-2.12.so

39f4a03000-39f4a04000 rw-p 00003000 08:01 10133                          /lib64/libdl-2.12.so

39f4c00000-39f4d89000 r-xp 00000000 08:01 10131                          /lib64/libc-2.12.so

39f4d89000-39f4f89000 ---p 00189000 08:01 10131                          /lib64/libc-2.12.so

39f4f89000-39f4f8d000 r--p 00189000 08:01 10131                          /lib64/libc-2.12.so

39f4f8d000-39f4f8e000 rw-p 0018d000 08:01 10131                          /lib64/libc-2.12.so

39f4f8e000-39f4f93000 rw-p 00000000 00:00 0

39f5000000-39f5017000 r-xp 00000000 08:01 1742                           /lib64/libpthread-2.12.so

39f5017000-39f5217000 ---p 00017000 08:01 1742                           /lib64/libpthread-2.12.so

39f5217000-39f5218000 r--p 00017000 08:01 1742                           /lib64/libpthread-2.12.so

39f5218000-39f5219000 rw-p 00018000 08:01 1742                           /lib64/libpthread-2.12.so

39f5219000-39f521d000 rw-p 00000000 00:00 0

39f5400000-39f5415000 r-xp 00000000 08:01 3622                           /lib64/libz.so.1.2.3

39f5415000-39f5614000 ---p 00015000 08:01 3622                           /lib64/libz.so.1.2.3

39f5614000-39f5615000 r--p 00014000 08:01 3622                           /lib64/libz.so.1.2.3

39f5615000-39f5616000 rw-p 00015000 08:01 3622                           /lib64/libz.so.1.2.3

39f5800000-39f5883000 r-xp 00000000 08:01 10147                          /lib64/libm-2.12.so

39f5883000-39f5a82000 ---p 00083000 08:01 10147                          /lib64/libm-2.12.so

39f5a82000-39f5a83000 r--p 00082000 08:01 10147                          /lib64/libm-2.12.so

39f5a83000-39f5a84000 rw-p 00083000 08:01 10147                          /lib64/libm-2.12.so

39f5c00000-39f5ce8000 r-xp 00000000 08:01 536101                         /usr/lib64/libstdc++.so.6.0.13

39f5ce8000-39f5ee8000 ---p 000e8000 08:01 536101                         /usr/lib64/libstdc++.so.6.0.13

39f5ee8000-39f5eef000 r--p 000e8000 08:01 536101                         /usr/lib64/libstdc++.so.6.0.13

39f5eef000-39f5ef1000 rw-p 000ef000 08:01 536101                         /usr/lib64/libstdc++.so.6.0.13

39f5ef1000-39f5f06000 rw-p 00000000 00:00 0

39f6c00000-39f6c22000 r-xp 00000000 08:01 1724                           /lib64/libncurses.so.5.7

39f6c22000-39f6e21000 ---p 00022000 08:01 1724                           /lib64/libncurses.so.5.7

39f6e21000-39f6e22000 rw-p 00021000 08:01 1724                           /lib64/libncurses.so.5.7

39f7000000-39f701d000 r-xp 00000000 08:01 1601                           /lib64/libtinfo.so.5.7

39f701d000-39f721d000 ---p 0001d000 08:01 1601                           /lib64/libtinfo.so.5.7

39f721d000-39f7221000 rw-p 0001d000 08:01 1601                           /lib64/libtinfo.so.5.7

39fbc00000-39fbc03000 r-xp 00000000 08:01 1736                           /lib64/libgpg-error.so.0.5.0

39fbc03000-39fbe02000 ---p 00003000 08:01 1736                           /lib64/libgpg-error.so.0.5.0

39fbe02000-39fbe03000 r--p 00002000 08:01 1736                           /lib64/libgpg-error.so.0.5.0

39fbe03000-39fbe04000 rw-p 00003000 08:01 1736                           /lib64/libgpg-error.so.0.5.0

39fc000000-39fc072000 r-xp 00000000 08:01 2252                           /lib64/libgcrypt.so.11.5.3

39fc072000-39fc271000 ---p 00072000 08:01 2252                           /lib64/libgcrypt.so.11.5.3

39fc271000-39fc272000 r--p 00071000 08:01 2252                           /lib64/libgcrypt.so.11.5.3

39fc272000-39fc275000 rw-p 00072000 08:01 2252                           /lib64/libgcrypt.so.11.5.3

7f2b64000000-7f2b64021000 rw-p 00000000 00:00 0

7f2b64021000-7f2b68000000 ---p 00000000 00:00 0

7f2b6b59f000-7f2b6b5a0000 ---p 00000000 00:00 0

7f2b6b5a0000-7f2b6bfa0000 rw-p 00000000 00:00 0

7f2b6bfa0000-7f2b6bfa1000 ---p 00000000 00:00 0

7f2b6bfa1000-7f2b6c9a1000 rw-p 00000000 00:00 0

7f2b6c9a1000-7f2b6c9a2000 ---p 00000000 00:00 0

7f2b6c9a2000-7f2b6d3a2000 rw-p 00000000 00:00 0

7f2b6d3a2000-7f2b6d3a3000 ---p 00000000 00:00 0

7f2b6d3a3000-7f2b6dda3000 rw-p 00000000 00:00 0

7f2b6dda3000-7f2b6dda4000 ---p 00000000 00:00 0

7f2b6dda4000-7f2b6e7a4000 rw-p 00000000 00:00 0

7f2b6e7a4000-7f2b6e7a5000 ---p 00000000 00:00 0

7f2b6e7a5000-7f2b6f1a5000 rw-p 00000000 00:00 0

7f2b6f1a5000-7f2b6f1a6000 ---p 00000000 00:00 0

7f2b6f1a6000-7f2b6fba6000 rw-p 00000000 00:00 0

7f2b6fba6000-7f2b6fba7000 ---p 00000000 00:00 0

7f2b6fba7000-7f2b705a7000 rw-p 00000000 00:00 0

7f2b705a7000-7f2b705a8000 ---p 00000000 00:00 0

7f2b705a8000-7f2b70fa8000 rw-p 00000000 00:00 0

7f2b70fa8000-7f2b70fa9000 ---p 00000000 00:00 0

7f2b70fa9000-7f2b719a9000 rw-p 00000000 00:00 0

7f2b719a9000-7f2b719aa000 ---p 00000000 00:00 0

7f2b719aa000-7f2b723aa000 rw-p 00000000 00:00 0

7f2b723aa000-7f2b723ab000 ---p 00000000 00:00 0

7f2b723ab000-7f2b72dab000 rw-p 00000000 00:00 0

7f2b72dab000-7f2b72dac000 ---p 00000000 00:00 0

7f2b72dac000-7f2b737ac000 rw-p 00000000 00:00 0

7f2b737ac000-7f2b737ad000 ---p 00000000 00:00 0

7f2b737ad000-7f2b741ad000 rw-p 00000000 00:00 0

7f2b741ad000-7f2b741ae000 ---p 00000000 00:00 0

7f2b741ae000-7f2b74bae000 rw-p 00000000 00:00 0

7f2b74bae000-7f2b74baf000 ---p 00000000 00:00 0

7f2b74baf000-7f2b755af000 rw-p 00000000 00:00 0

7f2b755af000-7f2b755b0000 ---p 00000000 00:00 0

7f2b755b0000-7f2b75fb0000 rw-p 00000000 00:00 0

7f2b75fb0000-7f2b75fb1000 ---p 00000000 00:00 0

7f2b75fb1000-7f2b769b1000 rw-p 00000000 00:00 0

7f2b769b1000-7f2b769b2000 ---p 00000000 00:00 0

(gdb) bt full

#0  0x00000039f4c328a5 in raise () from /lib64/libc.so.6

No symbol table info available.

#1  0x00000039f4c34085 in abort () from /lib64/libc.so.6

No symbol table info available.

#2  0x00000039f4c6ffe7 in __libc_message () from /lib64/libc.so.6

No symbol table info available.

#3  0x00000039f4c75916 in malloc_printerr () from /lib64/libc.so.6

No symbol table info available.

#4  0x00000039f4c78443 in _int_free () from /lib64/libc.so.6

No symbol table info available.

#5  0x00000000004ae2f5 in evbuffer_chain_free (buf=0x1782b70, len=<value optimized out>) at buffer.c:226

        __func__ = "evbuffer_chain_free"

#6  evbuffer_drain (buf=0x1782b70, len=<value optimized out>) at buffer.c:924

        chain = 0x7f2bac000e10

        next = 0x0

        remaining = <value optimized out>

        old_len = 960

        result = <value optimized out>

        __func__ = "evbuffer_drain"

#7  0x00000000004aeb2c in evbuffer_write_atmost (buffer=0x1782b70, fd=57, howmuch=960) at buffer.c:2314

        n = 960

#8  0x00000000004b06a5 in bufferevent_writecb (fd=57, event=<value optimized out>, arg=0x17828d0) at bufferevent_sock.c:260

        bufev = <value optimized out>

        bufev_p = <value optimized out>

        res = <value optimized out>

        what = <value optimized out>

        connected = 0

        atmost = 16384

#9  0x00000000004a8ec1 in event_process_active_single_queue (base=0x1775730, flags=0) at event.c:1308

No locals.

#10 event_process_active (base=0x1775730, flags=0) at event.c:1375

        i = 0

#11 event_base_loop (base=0x1775730, flags=0) at event.c:1572

        n = 2

        evsel = 0x553dc0

        tv = {tv_sec = 5, tv_usec = 0}

        tv_p = 0x0

        res = <value optimized out>

        retval = <value optimized out>

        __func__ = "event_base_loop"

#12 0x00000000004130b2 in ns_replicator (arg=0xce6ff0) at ns_replicator.c:1191

        rep = 0xa9b4c0

        pThr = 0xce6ff0

        __PRETTY_FUNCTION__ = "ns_replicator"

        __FUNCTION__ = "ns_replicator"

#13 0x00000039f5007851 in start_thread () from /lib64/libpthread.so.0

No symbol table info available.

#14 0x00000039f4ce811d in clone () from /lib64/libc.so.6

No symbol table info available.

I wonder why timer doesn't work well suddenly after I sent and received data.

I used libevent-2.0.21-stable version.

Below is my code.

/********* setup libevent *****************/

// init timer

rep->timer = evtimer_new(rep->evbase, _ns_rep_timer, rep);

evtimer_add(rep->timer, &rep->commit_interval);

// setup bufferevent

struct bufferevent *bev = bufferevent_socket_new(rep->evbase, sock, BEV_OPT_CLOSE_ON_FREE);

bufferevent_setwatermark(bev, EV_READ, sizeof(ns_rep_tcp_msg_header_t), 0);

bufferevent_setwatermark(bev, EV_WRITE, 0, 0);

bufferevent_setcb(bev, _ns_rep_tcp_read, _ns_rep_tcp_write, NULL, rep);

bufferevent_enable(bev, EV_READ|EV_WRITE);

/*************************************************/

timer is used to send a commit-message in commit_interval.

and bufferevent is used to read/write some data or read a commit_message.

/ ********** read callback ********/

static void _ns_rep_tcp_write(struct bufferevent *bev, void *arg)

{

................

       /* check the size of input buffer */

        /* just get a pointer of input buffer, not remove data from input buffer */

        input_buf = bufferevent_get_input(bev);

        input_buf_len = evbuffer_get_length(input_buf);

        if (input_buf_len < sizeof(ns_rep_tcp_msg_header_t)) {

            NO_TRACE("input_buf_len < sizeof(ns_rep_tcp_msg_header_t)(%d)", sizeof(ns_rep_tcp_msg_header_t));

            bufferevent_setwatermark(bev, EV_READ, sizeof(ns_rep_tcp_msg_header_t), 0);

            break;

        }

        /* just copyout data from input buffer to check body_size, not remove data from input buffer */

        NO_TRACE("evbuffer_copyout. input_buf_len:%d", input_buf_len);

        res = evbuffer_copyout(input_buf, &msg, sizeof(ns_rep_tcp_msg_header_t));

        if (res != sizeof(ns_rep_tcp_msg_header_t)) {

            NO_ERROR(NE_FATAL, "fail to evbuffer_copyout. res:%d", res);

            break;

        }

        /* check the header and body are all in input buffer */

        size_t size_with_body = sizeof(ns_rep_tcp_msg_header_t) + msg.body_size;

        if (input_buf_len < size_with_body) {

            NO_TRACE("input_buf_len < size_with_body(%d)", size_with_body);

            NO_TRACE("change read-low-watermark to size_with_body(%d", size_with_body);

            /* change watermark to read header and body in next invoked time */

            bufferevent_setwatermark(bev, EV_READ, size_with_body, 0);

            break;

        }

        // drain a header in input buffer because a header was copied out before.

        NO_TRACE("evbuffer_drain. cgid:%d", msg.cgid);

        res = evbuffer_drain(input_buf, sizeof(ns_rep_tcp_msg_header_t));

/************************************************************************************/

/*************** write callback ***********/

static void _ns_rep_tcp_write(struct bufferevent *bev, void *arg)

{

    // not used

}

/*********************************************/

/*************** timer *****************/

// timer for send COMMIT

static void _ns_rep_timer(evutil_socket_t fd, short events, void *arg)

{

    int i;

    for (i = 0 ; i < 2 ; i++) {

        bufferevent_write(bev[i], msg, sizeof(ns_rep_tcp_msg_header_t));

    }

    evtimer_add(rep->timer, &rep->commit_interval);

}

/*****************************************/

timer interval is 1 second.

Why a timer doesn't work well after some data is sent/received by bufferevent read/write callback?

please give me some advise.

thank you.