[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
Re: [Libevent-users] multithreading problem
I now tried setting up locking callbacks. But my programm still
crashes while stress testing.
I've attached the current code and a stack trace from gdb.
I don't know if it's an openssl or libevent related problem, but I
think I set the locking functions up correctly.
Regards
Björn
/* Program received signal SIGSEGV, Segmentation fault.
0x000000000040cb76 in bufferevent_setcb (bufev=0xfffffffff007b580,
readcb=0x40374e <readcb>, writecb=0, eventcb=0, cbarg=0x0)
at bufferevent.c:352
352 BEV_LOCK(bufev);
(gdb) bt
#0 0x000000000040cb76 in bufferevent_setcb (bufev=0xfffffffff007b580,
readcb=0x40374e <readcb>, writecb=0, eventcb=0, cbarg=0x0)
at bufferevent.c:352
#1 0x0000000000403878 in acceptcb ()
#2 0x000000000041d726 in listener_read_cb (fd=10, what=<value optimized out>,
p=<value optimized out>) at listener.c:412
#3 0x0000000000416c8a in event_process_active_single_queue (base=0x62f940,
activeq=0x62dc70, max_to_process=2147483647, endtime=<value optimized out>)
at event.c:1471
#4 0x00000000004174e7 in event_process_active (base=0x62f940,
flags=<value optimized out>) at event.c:1538
#5 event_base_loop (base=0x62f940, flags=<value optimized out>)
at event.c:1761
#6 0x0000000000403aed in main ()
*/
#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
#include <pthread.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#include <openssl/rand.h>
#include <event.h>
#include <event2/listener.h>
#include <event2/bufferevent_ssl.h>
#include <event2/thread.h>
#define NUM_THREADS 4
int next_thread = 0;
pthread_t thread[NUM_THREADS];
struct event_base *thread_base[NUM_THREADS];
static pthread_mutex_t *lock_cs;
static long *lock_count;
void pthreads_locking_callback(int mode, int type, char *file,
int line) {
if (mode & CRYPTO_LOCK) {
pthread_mutex_lock(&(lock_cs[type]));
lock_count[type]++;
}
else {
pthread_mutex_unlock(&(lock_cs[type]));
}
}
unsigned long pthreads_thread_id(void) {
unsigned long ret;
ret = (unsigned long)pthread_self();
return (ret);
}
struct CRYPTO_dynlock_value
{
pthread_mutex_t mutex;
};
static struct CRYPTO_dynlock_value *dyn_create_function(const char *file,
int line) {
struct CRYPTO_dynlock_value *value;
value = (struct CRYPTO_dynlock_value *)
malloc(sizeof(struct CRYPTO_dynlock_value));
if (!value) {
goto err;
}
pthread_mutex_init(&value->mutex, NULL);
return value;
err:
return (NULL);
}
static void dyn_lock_function(int mode, struct CRYPTO_dynlock_value *l,
const char *file, int line) {
if (mode & CRYPTO_LOCK) {
pthread_mutex_lock(&l->mutex);
} else {
pthread_mutex_unlock(&l->mutex);
}
}
static void dyn_destroy_function(struct CRYPTO_dynlock_value *l,
const char *file, int line) {
pthread_mutex_destroy(&l->mutex);
free(l);
}
void CRYPTO_thread_setup(void) {
int i;
lock_cs = OPENSSL_malloc(CRYPTO_num_locks() * sizeof(pthread_mutex_t));
lock_count = OPENSSL_malloc(CRYPTO_num_locks() * sizeof(long));
for (i = 0; i < CRYPTO_num_locks(); i++) {
lock_count[i] = 0;
pthread_mutex_init(&(lock_cs[i]), NULL);
}
CRYPTO_set_id_callback((unsigned long (*)())pthreads_thread_id);
CRYPTO_set_locking_callback((void (*)())pthreads_locking_callback);
CRYPTO_set_dynlock_create_callback(dyn_create_function);
CRYPTO_set_dynlock_lock_callback(dyn_lock_function);
CRYPTO_set_dynlock_destroy_callback(dyn_destroy_function);
}
void CRYPTO_thread_cleanup(void) {
int i;
CRYPTO_set_dynlock_create_callback(NULL);
CRYPTO_set_dynlock_lock_callback(NULL);
CRYPTO_set_dynlock_destroy_callback(NULL);
CRYPTO_set_locking_callback(NULL);
for (i = 0; i < CRYPTO_num_locks(); i++) {
pthread_mutex_destroy(&(lock_cs[i]));
}
OPENSSL_free(lock_cs);
OPENSSL_free(lock_count);
}
static void
readcb(struct bufferevent * bev, void *arg)
{
struct evbuffer *resp = evbuffer_new();
struct evbuffer* buf;
buf = evbuffer_new();
char *res = "HTTP/1.1 202 OK\r\n"
"Date: Wed, 28 Nov 2012 12:20:43 GMT\r\n"
"Server: Test\r\n"
"Connection: Keep-Alive\r\n"
"Keep-Alive: timeout=15, max=98\r\n"
"Content-Type: text/html\r\n"
"Vary: Accept-Encoding\r\n"
"Content-Length: 4\r\n"
"\r\n"
"test";
evbuffer_add(resp, res, strlen(res));
bufferevent_write_buffer(bev, resp);
evbuffer_free(resp);
}
static void acceptcb(struct evconnlistener *listener, int sock, struct
sockaddr *sa, int sa_len, void *arg) {
struct event_base *evbase;
struct bufferevent *bev;
SSL_CTX *server_ctx;
SSL *client_ctx;
server_ctx = (SSL_CTX *)arg;
client_ctx = SSL_new(server_ctx);
evbase = evconnlistener_get_base(listener);
next_thread = (next_thread + 1) % NUM_THREADS;
bev = bufferevent_openssl_socket_new(thread_base[next_thread],
sock, client_ctx,
BUFFEREVENT_SSL_ACCEPTING,
BEV_OPT_CLOSE_ON_FREE|BEV_OPT_THREADSAFE|BEV_OPT_DEFER_CALLBACKS);
bufferevent_setcb(bev, readcb, NULL, NULL, NULL);
bufferevent_enable(bev, EV_READ);
}
static void ignore_sigpipe() {
struct sigaction sa;
memset(&sa, 0, sizeof(sa));
sa.sa_handler = SIG_IGN;
if (sigemptyset(&sa.sa_mask) < 0 || sigaction(SIGPIPE, &sa, 0) < 0) {
printf("Could not ignore the SIGPIPE signal");
exit(EXIT_FAILURE);
}
}
static SSL_CTX *ssl_init() {
SSL_CTX *ctx;
SSL_library_init();
CRYPTO_thread_setup();
ctx = SSL_CTX_new(SSLv23_server_method());
if ( !SSL_CTX_use_certificate_file(ctx, "crt.pem", SSL_FILETYPE_PEM) )
return NULL;
if ( !SSL_CTX_use_PrivateKey_file(ctx, "crt.key", SSL_FILETYPE_PEM) )
return NULL;
SSL_CTX_set_verify(ctx, SSL_VERIFY_NONE, NULL);
return ctx;
}
void *worker_function(void *ptr) {
struct event_base **evbase = (struct event_base **)ptr;
ignore_sigpipe();
*evbase = event_base_new();
event_base_loop(*evbase, EVLOOP_NO_EXIT_ON_EMPTY);
return NULL;
}
int main(int arc, char **argv) {
SSL_CTX *ctx;
struct evconnlistener *listener;
struct event_base *evbase;
struct sockaddr_in sin;
memset(&sin, 0, sizeof(sin));
sin.sin_family = AF_INET;
sin.sin_port = htons(443);
sin.sin_addr.s_addr = htonl(INADDR_ANY);
ignore_sigpipe();
ctx = ssl_init();
if ( ctx == NULL )
return EXIT_FAILURE;
evthread_use_pthreads();
event_enable_debug_logging(EVENT_DBG_ALL);
evthread_enable_lock_debuging();
evbase = event_base_new();
listener = evconnlistener_new_bind(
evbase, acceptcb, (void *)ctx,
LEV_OPT_THREADSAFE | LEV_OPT_CLOSE_ON_FREE |
LEV_OPT_REUSEABLE, 1024,
(struct sockaddr *)&sin, sizeof(sin));
int i;
for (i = 0; i < NUM_THREADS; i++) {
pthread_create(&thread[i], NULL, worker_function, (void
*)&thread_base[i]);
}
event_base_loop(evbase, 0);
return EXIT_SUCCESS;
}
2012/12/18 Mark Ellzey <mthomas@xxxxxxxxxx>:
> On Tue, Dec 18, 2012 at 12:53:11AM +0100, Bj?rn K. wrote:
>> I tried to simplify my code as much as possible. You can see the
>> simplified code and the error message from gdb below.
>> Every thread has its own event_base. The only access to the
>> event_bases of the threads is from the event_base_loop of the
>> corresponding thread and the bufferevent functions in acceptcb().
>> The basic idea was that acceptcb() assigns every new connection to one
>> of the threads.
>
>
> OpenSSL is not thread-safe. You need to setup your ssl locking
> callbacks.
> ***********************************************************************
> To unsubscribe, send an e-mail to majordomo@xxxxxxxxxxxxx with
> unsubscribe libevent-users in the body.
***********************************************************************
To unsubscribe, send an e-mail to majordomo@xxxxxxxxxxxxx with
unsubscribe libevent-users in the body.