[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

Re: [Libevent-users] multithreading problem



I now tried setting up locking callbacks. But my programm still
crashes while stress testing.
I've attached the current code and a stack trace from gdb.
I don't know if it's an openssl or libevent related problem, but I
think I set the locking functions up correctly.

Regards
Björn


/* Program received signal SIGSEGV, Segmentation fault.
0x000000000040cb76 in bufferevent_setcb (bufev=0xfffffffff007b580,
    readcb=0x40374e <readcb>, writecb=0, eventcb=0, cbarg=0x0)
    at bufferevent.c:352
352		BEV_LOCK(bufev);
(gdb) bt
#0  0x000000000040cb76 in bufferevent_setcb (bufev=0xfffffffff007b580,
    readcb=0x40374e <readcb>, writecb=0, eventcb=0, cbarg=0x0)
    at bufferevent.c:352
#1  0x0000000000403878 in acceptcb ()
#2  0x000000000041d726 in listener_read_cb (fd=10, what=<value optimized out>,
    p=<value optimized out>) at listener.c:412
#3  0x0000000000416c8a in event_process_active_single_queue (base=0x62f940,
    activeq=0x62dc70, max_to_process=2147483647, endtime=<value optimized out>)
    at event.c:1471
#4  0x00000000004174e7 in event_process_active (base=0x62f940,
    flags=<value optimized out>) at event.c:1538
#5  event_base_loop (base=0x62f940, flags=<value optimized out>)
    at event.c:1761
#6  0x0000000000403aed in main ()
 */

#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
#include <pthread.h>

#include <openssl/ssl.h>
#include <openssl/err.h>
#include <openssl/rand.h>

#include <event.h>
#include <event2/listener.h>
#include <event2/bufferevent_ssl.h>
#include <event2/thread.h>

#define NUM_THREADS 4

int next_thread = 0;
pthread_t thread[NUM_THREADS];
struct event_base *thread_base[NUM_THREADS];

static pthread_mutex_t *lock_cs;
static long *lock_count;

void pthreads_locking_callback(int mode, int type, char *file,
                               int line) {

    if (mode & CRYPTO_LOCK) {
        pthread_mutex_lock(&(lock_cs[type]));
        lock_count[type]++;
    }
    else {
        pthread_mutex_unlock(&(lock_cs[type]));
    }
}

unsigned long pthreads_thread_id(void) {

    unsigned long ret;

    ret = (unsigned long)pthread_self();
    return (ret);

}

struct CRYPTO_dynlock_value
{
     pthread_mutex_t mutex;
};

static struct CRYPTO_dynlock_value *dyn_create_function(const char *file,
int line) {
     struct CRYPTO_dynlock_value *value;

     value = (struct CRYPTO_dynlock_value *)
         malloc(sizeof(struct CRYPTO_dynlock_value));
     if (!value) {
         goto err;
     }
     pthread_mutex_init(&value->mutex, NULL);

     return value;

   err:
     return (NULL);
}

static void dyn_lock_function(int mode, struct CRYPTO_dynlock_value *l,
                               const char *file, int line) {
     if (mode & CRYPTO_LOCK) {
         pthread_mutex_lock(&l->mutex);
     } else {
         pthread_mutex_unlock(&l->mutex);
     }
}

static void dyn_destroy_function(struct CRYPTO_dynlock_value *l,
                                  const char *file, int line) {
     pthread_mutex_destroy(&l->mutex);
     free(l);
}

void CRYPTO_thread_setup(void) {

    int i;

    lock_cs = OPENSSL_malloc(CRYPTO_num_locks() * sizeof(pthread_mutex_t));
    lock_count = OPENSSL_malloc(CRYPTO_num_locks() * sizeof(long));

    for (i = 0; i < CRYPTO_num_locks(); i++) {
        lock_count[i] = 0;
        pthread_mutex_init(&(lock_cs[i]), NULL);
    }

    CRYPTO_set_id_callback((unsigned long (*)())pthreads_thread_id);
    CRYPTO_set_locking_callback((void (*)())pthreads_locking_callback);
    CRYPTO_set_dynlock_create_callback(dyn_create_function);
    CRYPTO_set_dynlock_lock_callback(dyn_lock_function);
    CRYPTO_set_dynlock_destroy_callback(dyn_destroy_function);

}

void CRYPTO_thread_cleanup(void) {

    int i;

    CRYPTO_set_dynlock_create_callback(NULL);
    CRYPTO_set_dynlock_lock_callback(NULL);
    CRYPTO_set_dynlock_destroy_callback(NULL);
    CRYPTO_set_locking_callback(NULL);

    for (i = 0; i < CRYPTO_num_locks(); i++) {
        pthread_mutex_destroy(&(lock_cs[i]));
    }

    OPENSSL_free(lock_cs);
    OPENSSL_free(lock_count);

}

static void
readcb(struct bufferevent * bev, void *arg)
{

    struct evbuffer *resp = evbuffer_new();

    struct evbuffer* buf;
    buf = evbuffer_new();

    char *res = "HTTP/1.1 202 OK\r\n"
                "Date: Wed, 28 Nov 2012 12:20:43 GMT\r\n"
                "Server: Test\r\n"
                "Connection: Keep-Alive\r\n"
                "Keep-Alive: timeout=15, max=98\r\n"
                "Content-Type: text/html\r\n"
                "Vary: Accept-Encoding\r\n"
                "Content-Length: 4\r\n"
                "\r\n"
                "test";
    evbuffer_add(resp, res, strlen(res));
    bufferevent_write_buffer(bev, resp);
    evbuffer_free(resp);

}

static void acceptcb(struct evconnlistener *listener, int sock, struct
sockaddr *sa, int sa_len, void *arg) {

    struct event_base *evbase;
    struct bufferevent *bev;
    SSL_CTX *server_ctx;
    SSL *client_ctx;

    server_ctx = (SSL_CTX *)arg;
    client_ctx = SSL_new(server_ctx);
    evbase = evconnlistener_get_base(listener);

    next_thread = (next_thread + 1) % NUM_THREADS;

    bev = bufferevent_openssl_socket_new(thread_base[next_thread],
sock, client_ctx,
                                         BUFFEREVENT_SSL_ACCEPTING,

BEV_OPT_CLOSE_ON_FREE|BEV_OPT_THREADSAFE|BEV_OPT_DEFER_CALLBACKS);

    bufferevent_setcb(bev, readcb, NULL, NULL, NULL);
    bufferevent_enable(bev, EV_READ);

}

static void ignore_sigpipe() {

    struct sigaction sa;

    memset(&sa, 0, sizeof(sa));
    sa.sa_handler = SIG_IGN;

    if (sigemptyset(&sa.sa_mask) < 0 || sigaction(SIGPIPE, &sa, 0) < 0) {
        printf("Could not ignore the SIGPIPE signal");
        exit(EXIT_FAILURE);
    }

}

static SSL_CTX *ssl_init() {

    SSL_CTX *ctx;

    SSL_library_init();
    CRYPTO_thread_setup();

    ctx = SSL_CTX_new(SSLv23_server_method());

    if ( !SSL_CTX_use_certificate_file(ctx, "crt.pem", SSL_FILETYPE_PEM) )
        return NULL;

    if ( !SSL_CTX_use_PrivateKey_file(ctx, "crt.key", SSL_FILETYPE_PEM) )
        return NULL;

    SSL_CTX_set_verify(ctx, SSL_VERIFY_NONE, NULL);

    return ctx;

}


void *worker_function(void *ptr) {

    struct event_base **evbase = (struct event_base **)ptr;
    ignore_sigpipe();
    *evbase = event_base_new();

    event_base_loop(*evbase, EVLOOP_NO_EXIT_ON_EMPTY);

    return NULL;

}

int main(int arc, char **argv) {

    SSL_CTX *ctx;
    struct evconnlistener *listener;
    struct event_base *evbase;
    struct sockaddr_in sin;

    memset(&sin, 0, sizeof(sin));
    sin.sin_family = AF_INET;
    sin.sin_port = htons(443);
    sin.sin_addr.s_addr = htonl(INADDR_ANY);

    ignore_sigpipe();

    ctx = ssl_init();
    if ( ctx == NULL )
        return EXIT_FAILURE;

    evthread_use_pthreads();
	event_enable_debug_logging(EVENT_DBG_ALL);
	evthread_enable_lock_debuging();
    evbase = event_base_new();


    listener = evconnlistener_new_bind(
                        evbase, acceptcb, (void *)ctx,
                        LEV_OPT_THREADSAFE | LEV_OPT_CLOSE_ON_FREE |
LEV_OPT_REUSEABLE, 1024,
                        (struct sockaddr *)&sin, sizeof(sin));


    int i;
    for (i = 0; i < NUM_THREADS; i++) {
        pthread_create(&thread[i], NULL, worker_function, (void
*)&thread_base[i]);
    }


    event_base_loop(evbase, 0);

    return EXIT_SUCCESS;

}


2012/12/18 Mark Ellzey <mthomas@xxxxxxxxxx>:
> On Tue, Dec 18, 2012 at 12:53:11AM +0100, Bj?rn K. wrote:
>> I tried to simplify my code as much as possible. You can see the
>> simplified code and the error message from gdb below.
>> Every thread has its own event_base. The only access to the
>> event_bases of the threads is from the event_base_loop of the
>> corresponding thread and the bufferevent functions in acceptcb().
>> The basic idea was that acceptcb() assigns every new connection to one
>> of the threads.
>
>
> OpenSSL is not thread-safe. You need to setup your ssl locking
> callbacks.
> ***********************************************************************
> To unsubscribe, send an e-mail to majordomo@xxxxxxxxxxxxx with
> unsubscribe libevent-users    in the body.
***********************************************************************
To unsubscribe, send an e-mail to majordomo@xxxxxxxxxxxxx with
unsubscribe libevent-users    in the body.