Hello,
while using libevent for some benchmarking tests I have encountered
an issue that I cant tell if its a bug or a feature.
The attempt to call select() on more than 1024 descriptors crashes
on Ubuntu server and OpenSolaris with buffer overflow,
while on Centos and FreeBSD it can hanlde them.
I know select is not supposed to work with that amount of fd's but
then why it does so selectively for some platforms.
For the setup I use a slightly modified http-server code from
libevent samples (modified with that it binds to a certain port,
and has additional command line parameter to choose the event
dispatch mechanism) and the client just spawns a requested number of
idle
open sockets to the server port, without transferring any data. The
hardware is the same for all OS'es, and in all cases the OS is kept
as
default as possible (the limits on open files are set to be higher,
no firewall and no selinux).
So I wonder if libevent has to do anything with this, or is it OS feature?
I tried to change the __FD_SETSIZE in
/usr/include/x86_64-linux-gnu/bits/typesizes.h (the select.c seems
to use this value)
and recompile libevent and server code but with no effect.
A comment in libevent's select.c shows that something similar was
already there:
#ifdef __APPLE__
/* Apple wants us to define this if we might ever pass more than
* FD_SETSIZE bits to select(). */
#define _DARWIN_UNLIMITED_SELECT
#endif
Could it be used on platforms other than Apple?
I would appreciate any opinion or explanation on that issue.
Thank you.
Kind Regards,
Roman Florea.
------- The output---
Running on FreeBSD: truss bin/httpserver /tmp -select
...
accept(5,{ AF_INET 130.230.141.41:5313 },0x7fffffffd82c) = 2003 (0x7d3)
fcntl(2003,F_SETFD,FD_CLOEXEC) = 0 (0x0)
fcntl(2003,F_SETFL,O_NONBLOCK) = 0 (0x0)
accept(5,{ AF_INET 130.230.141.41:5314 },0x7fffffffd82c) = 2004 (0x7d4)
fcntl(2004,F_SETFD,FD_CLOEXEC) = 0 (0x0)
fcntl(2004,F_SETFL,O_NONBLOCK) = 0 (0x0)
accept(5,{ AF_INET 130.230.141.41:5315 },0x7fffffffd82c) = 2005 (0x7d5)
fcntl(2005,F_SETFD,FD_CLOEXEC) = 0 (0x0)
fcntl(2005,F_SETFL,O_NONBLOCK) = 0 (0x0)
accept(5,0x7fffffffd7a0,0x7fffffffd82c) ERR#35 'Resource
temporarily unavailable'
select(2006,{5 6 7 8 9 10 11 12 13 14 --CUT OUTPUT OF CONSECUTIVE
FD NUMBERS--
1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996
1997 1998 1999 2000 2001 2002 2003 2004 2005},{},0x0,0x0) = 1 (0x1)
clock_gettime(4,{5765029.847206292 }) = 0 (0x0)
gettimeofday({1370160860.074904 },0x0) = 0 (0x0)
accept(5,{ AF_INET 130.230.141.45:34819 },0x7fffffffd82c) = 2006 (0x7d6)
...
The result on Ubuntu: strace bin/httpserver /tmp -select
...
accept4(5, 0x7fff87a677d0, [128], SOCK_CLOEXEC|SOCK_NONBLOCK) = -1
EAGAIN (Resource temporarily unavailable)
select(1023, [5 6 7 8 9 10 11 12 13 14 15 16 17 18 --CUT OUTPUT OF
CONSECUTIVE FD NUMBERS-- 1019 1020 1021 1022], [], NULL, NULL) = 1
(in [5])
accept4(5, {sa_family=AF_INET, sin_port=htons(26236),
sin_addr=inet_addr("192.168.141.41")}, [16],
SOCK_CLOEXEC|SOCK_NONBLOCK) = 1023
accept4(5, {sa_family=AF_INET, sin_port=htons(26237),
sin_addr=inet_addr("192.168.141.41")}, [16],
SOCK_CLOEXEC|SOCK_NONBLOCK) = 1024
open("/dev/tty", O_RDWR|O_NOCTTY|O_NONBLOCK) = 1025
writev(1025, [{"*** ", 4}, {"buffer overflow detected", 24}, {" ***:
", 6}, {"bin/httpserver", 14}, {" terminated\n", 12}], 5*** buffer
overflow detected ***: bin/httpserver terminated
) = 60
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
-1, 0) = 0x7f4ce5a06000
open("/opt/tlt-bench/lib/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = -1
ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 1026
fstat(1026, {st_mode=S_IFREG|0644, st_size=19195, ...}) = 0
mmap(NULL, 19195, PROT_READ, MAP_PRIVATE, 1026, 0) = 0x7f4ce59fa000
close(1026) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
directory)
open("/lib/x86_64-linux-gnu/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 1026
read(1026,
"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20*\0\0\0\0\0\0"...,
832) = 832
fstat(1026, {st_mode=S_IFREG|0644, st_size=88400, ...}) = 0
mmap(NULL, 2184216, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE,
1026, 0) = 0x7f4ce4b9f000
mprotect(0x7f4ce4bb4000, 2093056, PROT_NONE) = 0
mmap(0x7f4ce4db3000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 1026, 0x14000) = 0x7f4ce4db3000
close(1026) = 0
mprotect(0x7f4ce4db3000, 4096, PROT_READ) = 0
munmap(0x7f4ce59fa000, 19195) = 0
futex(0x7f4ce5596e10, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f4ce4db41a4, FUTEX_WAKE_PRIVATE, 2147483647) = 0
write(1025, "======= Backtrace: =========\n", 29======= Backtrace: =========
) = 29
writev(1025, [{"/lib/x86_64-linux-gnu/libc.so.6", 31}, {"(", 1},
{"__fortify_fail", 14}, {"+0x", 3}, {"5c", 2}, {")", 1}, {"[0x", 3},
{"7f4ce52e482c", 12}, {"]\n", 2}],
9/lib/x86_64-linux-gnu/libc.so.6(__fortify_fail+0x5c)[0x7f4ce52e482c]
) = 69
writev(1025, [{"/lib/x86_64-linux-gnu/libc.so.6", 31}, {"(", 1},
{"+0x", 3}, {"109700", 6}, {")", 1}, {"[0x", 3}, {"7f4ce52e3700",
12}, {"]\n", 2}],
8/lib/x86_64-linux-gnu/libc.so.6(+0x109700)[0x7f4ce52e3700]
) = 59
writev(1025, [{"/lib/x86_64-linux-gnu/libc.so.6", 31}, {"(", 1},
{"+0x", 3}, {"10a7be", 6}, {")", 1}, {"[0x", 3}, {"7f4ce52e47be",
12}, {"]\n", 2}],
8/lib/x86_64-linux-gnu/libc.so.6(+0x10a7be)[0x7f4ce52e47be]
) = 59
writev(1025, [{"/opt/tlt-bench/lib/libevent-2.1."..., 36}, {"(", 1},
{"+0x", 3}, {"27b0e", 5}, {")", 1}, {"[0x", 3}, {"7f4ce55c0b0e",
12}, {"]\n", 2}],
8/opt/tlt-bench/lib/libevent-2.1.so.3(+0x27b0e)[0x7f4ce55c0b0e]
) = 63
writev(1025, [{"/opt/tlt-bench/lib/libevent-2.1."..., 36}, {"(", 1},
{"evmap_io_add_", 13}, {"+0x", 3}, {"1aa", 3}, {")", 1}, {"[0x", 3},
{"7f4ce55bab9a", 12}, {"]\n", 2}],
9/opt/tlt-bench/lib/libevent-2.1.so.3(evmap_io_add_+0x1aa)[0x7f4ce55bab9a]
) = 74
writev(1025, [{"/opt/tlt-bench/lib/libevent-2.1."..., 36}, {"(", 1},
{"event_add_nolock_", 17}, {"+0x", 3}, {"64e", 3}, {")", 1}, {"[0x",
3}, {"7f4ce55b5fae", 12}, {"]\n", 2}],
9/opt/tlt-bench/lib/libevent-2.1.so.3(event_add_nolock_+0x64e)[0x7f4ce55b5fae]
) = 78
writev(1025, [{"/opt/tlt-bench/lib/libevent-2.1."..., 36}, {"(", 1},
{"event_add", 9}, {"+0x", 3}, {"3a", 2}, {")", 1}, {"[0x", 3},
{"7f4ce55b653a", 12}, {"]\n", 2}],
9/opt/tlt-bench/lib/libevent-2.1.so.3(event_add+0x3a)[0x7f4ce55b653a]
) = 69
writev(1025, [{"/opt/tlt-bench/lib/libevent-2.1."..., 36}, {"(", 1},
{"+0x", 3}, {"19213", 5}, {")", 1}, {"[0x", 3}, {"7f4ce55b2213",
12}, {"]\n", 2}],
8/opt/tlt-bench/lib/libevent-2.1.so.3(+0x19213)[0x7f4ce55b2213]
) = 63
writev(1025, [{"/opt/tlt-bench/lib/libevent-2.1."..., 36}, {"(", 1},
{"bufferevent_enable", 18}, {"+0x", 3}, {"5d", 2}, {")", 1}, {"[0x",
3}, {"7f4ce55adbcd", 12}, {"]\n", 2}],
9/opt/tlt-bench/lib/libevent-2.1.so.3(bufferevent_enable+0x5d)[0x7f4ce55adbcd]
) = 78
writev(1025, [{"/opt/tlt-bench/lib/libevent-2.1."..., 36}, {"(", 1},
{"+0x", 3}, {"19144", 5}, {")", 1}, {"[0x", 3}, {"7f4ce55b2144",
12}, {"]\n", 2}],
8/opt/tlt-bench/lib/libevent-2.1.so.3(+0x19144)[0x7f4ce55b2144]
) = 63
writev(1025, [{"/opt/tlt-bench/lib/libevent-2.1."..., 36}, {"(", 1},
{"bufferevent_setfd", 17}, {"+0x", 3}, {"3b", 2}, {")", 1}, {"[0x",
3}, {"7f4ce55ae20b", 12}, {"]\n", 2}],
9/opt/tlt-bench/lib/libevent-2.1.so.3(bufferevent_setfd+0x3b)[0x7f4ce55ae20b]
) = 77
writev(1025, [{"/opt/tlt-bench/lib/libevent-2.1."..., 36}, {"(", 1},
{"+0x", 3}, {"38c53", 5}, {")", 1}, {"[0x", 3}, {"7f4ce55d1c53",
12}, {"]\n", 2}],
8/opt/tlt-bench/lib/libevent-2.1.so.3(+0x38c53)[0x7f4ce55d1c53]
) = 63
writev(1025, [{"/opt/tlt-bench/lib/libevent-2.1."..., 36}, {"(", 1},
{"+0x", 3}, {"267bf", 5}, {")", 1}, {"[0x", 3}, {"7f4ce55bf7bf",
12}, {"]\n", 2}],
8/opt/tlt-bench/lib/libevent-2.1.so.3(+0x267bf)[0x7f4ce55bf7bf]
) = 63
writev(1025, [{"/opt/tlt-bench/lib/libevent-2.1."..., 36}, {"(", 1},
{"+0x", 3}, {"1db09", 5}, {")", 1}, {"[0x", 3}, {"7f4ce55b6b09",
12}, {"]\n", 2}],
8/opt/tlt-bench/lib/libevent-2.1.so.3(+0x1db09)[0x7f4ce55b6b09]
) = 63
writev(1025, [{"/opt/tlt-bench/lib/libevent-2.1."..., 36}, {"(", 1},
{"event_base_loop", 15}, {"+0x", 3}, {"407", 3}, {")", 1}, {"[0x",
3}, {"7f4ce55b73f7", 12}, {"]\n", 2}],
9/opt/tlt-bench/lib/libevent-2.1.so.3(event_base_loop+0x407)[0x7f4ce55b73f7]
) = 76
writev(1025, [{"bin/httpserver", 14}, {"[0x", 3}, {"4025d8", 6},
{"]\n", 2}], 4bin/httpserver[0x4025d8]
) = 25
writev(1025, [{"/lib/x86_64-linux-gnu/libc.so.6", 31}, {"(", 1},
{"__libc_start_main", 17}, {"+0x", 3}, {"ed", 2}, {")", 1}, {"[0x",
3}, {"7f4ce51fb76d", 12}, {"]\n", 2}],
9/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xed)[0x7f4ce51fb76d]
) = 72
writev(1025, [{"bin/httpserver", 14}, {"[0x", 3}, {"4017a9", 6},
{"]\n", 2}], 4bin/httpserver[0x4017a9]
) = 25
write(1025, "======= Memory map: ========\n", 29======= Memory map: ========
) = 29
open("/proc/self/maps", O_RDONLY) = 1026
read(1026, "00400000-00403000 r-xp 00000000 "..., 1024) = 1024
write(1025, "00400000-00403000 r-xp 00000000 "...,
102400400000-00403000 r-xp 00000000 fc:00 150666
/opt/tlt-bench/bin/httpserver
00602000-00603000 r--p 00002000 fc:00 150666
/opt/tlt-bench/bin/httpserver
00603000-00604000 rw-p 00003000 fc:00 150666
/opt/tlt-bench/bin/httpserver
00803000-009f2000 rw-p 00000000 00:00 0
[heap]
7f4ce4b9f000-7f4ce4bb4000 r-xp 00000000 fc:00 1010
/lib/x86_64-linux-gnu/libgcc_s.so.1
7f4ce4bb4000-7f4ce4db3000 ---p 00015000 fc:00 1010
/lib/x86_64-linux-gnu/libgcc_s.so.1
7f4ce4db3000-7f4ce4db4000 r--p 00014000 fc:00 1010
/lib/x86_64-linux-gnu/libgcc_s.so.1
7f4ce4db4000-7f4ce4db5000 rw-p 00015000 fc:00 1010
/lib/x86_64-linux-gnu/libgcc_s.so.1
7f4ce4db5000-7f4ce4dcd000 r-xp 00000000 fc:00 1059
/lib/x86_64-linux-gnu/libpthread-2.15.so
7f4ce4dcd000-7f4ce4fcc000 ---p 00018000 fc:00 1059
/lib/x86_64-) = 1024
read(1026, "linux-gnu/libpthread-2.15.so\n7f4"..., 1024) = 1024
write(1025, "linux-gnu/libpthread-2.15.so\n7f4"...,
1024linux-gnu/libpthread-2.15.so
7f4ce4fcc000-7f4ce4fcd000 r--p 00017000 fc:00 1059
/lib/x86_64-linux-gnu/libpthread-2.15.so
7f4ce4fcd000-7f4ce4fce000 rw-p 00018000 fc:00 1059
/lib/x86_64-linux-gnu/libpthread-2.15.so
7f4ce4fce000-7f4ce4fd2000 rw-p 00000000 00:00 0
7f4ce4fd2000-7f4ce4fd9000 r-xp 00000000 fc:00 1065
/lib/x86_64-linux-gnu/librt-2.15.so
7f4ce4fd9000-7f4ce51d8000 ---p 00007000 fc:00 1065
/lib/x86_64-linux-gnu/librt-2.15.so
7f4ce51d8000-7f4ce51d9000 r--p 00006000 fc:00 1065
/lib/x86_64-linux-gnu/librt-2.15.so
7f4ce51d9000-7f4ce51da000 rw-p 00007000 fc:00 1065
/lib/x86_64-linux-gnu/librt-2.15.so
7f4ce51da000-7f4ce538f000 r-xp 00000000 fc:00 990
/lib/x86_64-linux-gnu/libc-2.15.so
7f4ce538f000-7f4ce558e000 ---p 001b5000 fc:00 990
/lib/x86_64-linux-gnu/libc-2.15.so
7f4ce558e000-7f4ce5592000 r--p 001b4000 fc:00 990 ) = 1024
read(1026, " /lib/x86_64-linux-gnu/lib"..., 1024) = 1024
write(1025, " /lib/x86_64-linux-gnu/lib"..., 1024
/lib/x86_64-linux-gnu/libc-2.15.so
7f4ce5592000-7f4ce5594000 rw-p 001b8000 fc:00 990
/lib/x86_64-linux-gnu/libc-2.15.so
7f4ce5594000-7f4ce5599000 rw-p 00000000 00:00 0
7f4ce5599000-7f4ce55e5000 r-xp 00000000 fc:00 1276
/opt/tlt-bench/lib/libevent-2.1.so.3.0.0
7f4ce55e5000-7f4ce57e5000 ---p 0004c000 fc:00 1276
/opt/tlt-bench/lib/libevent-2.1.so.3.0.0
7f4ce57e5000-7f4ce57e6000 r--p 0004c000 fc:00 1276
/opt/tlt-bench/lib/libevent-2.1.so.3.0.0
7f4ce57e6000-7f4ce57e7000 rw-p 0004d000 fc:00 1276
/opt/tlt-bench/lib/libevent-2.1.so.3.0.0
7f4ce57e7000-7f4ce57e8000 rw-p 00000000 00:00 0
7f4ce57e8000-7f4ce580a000 r-xp 00000000 fc:00 972
/lib/x86_64-linux-gnu/ld-2.15.so
7f4ce59ff000-7f4ce5a03000 rw-p 00000000 00:00 0
7f4ce5a06000-7f4ce5a0a000 rw-p 00000000 00:00 0
7f4ce5a0a000-7f4ce5a0b000 r--p 00022000 fc:00 972
/lib/x86_64-linux-gnu/ld-2.15.so
7f4ce5a0b0) = 1024
read(1026, "00-7f4ce5a0d000 rw-p 00023000 fc"..., 1024) = 341
write(1025, "00-7f4ce5a0d000 rw-p 00023000 fc"...,
34100-7f4ce5a0d000 rw-p 00023000 fc:00 972
/lib/x86_64-linux-gnu/ld-2.15.so
7fff87a49000-7fff87a6a000 rw-p 00000000 00:00 0
[stack]
7fff87add000-7fff87ade000 r-xp 00000000 00:00 0
[vdso]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0
[vsyscall]
) = 341
read(1026, "", 1024) = 0
close(1026) = 0
rt_sigprocmask(SIG_UNBLOCK, [ABRT], NULL, 8) = 0
tgkill(31662, 31662, SIGABRT) = 0
--- SIGABRT (Aborted) @ 0 (0) ---
+++ killed by SIGABRT (core dumped) +++
Aborted (core dumped)
***********************************************************************
To unsubscribe, send an e-mail to majordomo@xxxxxxxxxxxxx with
unsubscribe libevent-users in the body.