Re: [Libevent-users] multithreading problem

[Björn K. <bjoernk2@xxxxxxxxxxxxxx>]

I tried to simplify my code as much as possible. You can see the
simplified code and the error message from gdb below.
Every thread has its own event_base. The only access to the
event_bases of the threads is from the event_base_loop of the
corresponding thread and the bufferevent functions in acceptcb().
The basic idea was that acceptcb() assigns every new connection to one
of the threads.

An easy way to reproduce the error is using a webbrowser and keep F5 pressed.

Regards
Björn

// error:
// Program received signal SIGSEGV, Segmentation fault.
// 0x000000000040c520 in evthread_use_pthreads () at evthread_pthread.c:180
// 180		};

#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
#include <pthread.h>

#include <openssl/ssl.h>
#include <openssl/err.h>
#include <openssl/rand.h>

#include <event.h>
#include <event2/listener.h>
#include <event2/bufferevent_ssl.h>
#include <event2/thread.h>

#define NUM_THREADS 4

int next_thread = 0;
pthread_t thread[NUM_THREADS];
struct event_base *thread_base[NUM_THREADS];

static void
readcb(struct bufferevent * bev, void *arg)
{

    struct evbuffer *resp = evbuffer_new();

    struct evbuffer* buf;
    buf = evbuffer_new();

    char *res = "HTTP/1.1 202 OK\r\n"
                "Date: Wed, 28 Nov 2012 12:20:43 GMT\r\n"
                "Server: Test\r\n"
                "Connection: Keep-Alive\r\n"
                "Keep-Alive: timeout=15, max=98\r\n"
                "Content-Type: text/html\r\n"
                "Vary: Accept-Encoding\r\n"
                "Content-Length: 4\r\n"
                "\r\n"
                "test";
    evbuffer_add(resp, res, strlen(res));
    bufferevent_write_buffer(bev, resp);
    evbuffer_free(resp);

}

static void acceptcb(struct evconnlistener *listener, int sock, struct
sockaddr *sa, int sa_len, void *arg) {

    struct event_base *evbase;
    struct bufferevent *bev;
    SSL_CTX *server_ctx;
    SSL *client_ctx;

    server_ctx = (SSL_CTX *)arg;
    client_ctx = SSL_new(server_ctx);
    evbase = evconnlistener_get_base(listener);

    next_thread = (next_thread + 1) % NUM_THREADS;

    bev = bufferevent_openssl_socket_new(thread_base[next_thread],
sock, client_ctx,
                                         BUFFEREVENT_SSL_ACCEPTING,

BEV_OPT_CLOSE_ON_FREE|BEV_OPT_THREADSAFE);

    bufferevent_setcb(bev, readcb, NULL, NULL, NULL);
    bufferevent_enable(bev, EV_READ);


}

static void ignore_sigpipe() {

    struct sigaction sa;

    memset(&sa, 0, sizeof(sa));
    sa.sa_handler = SIG_IGN;

    if (sigemptyset(&sa.sa_mask) < 0 || sigaction(SIGPIPE, &sa, 0) < 0) {
        exit(EXIT_FAILURE);
    }

}

static SSL_CTX *ssl_init() {

    SSL_CTX *ctx;

    SSL_library_init();

    ctx = SSL_CTX_new(SSLv23_server_method());

    if ( !SSL_CTX_use_certificate_file(ctx, "crt.pem", SSL_FILETYPE_PEM) )
        return NULL;

    if ( !SSL_CTX_use_PrivateKey_file(ctx, "crt.key", SSL_FILETYPE_PEM) )
        return NULL;

    SSL_CTX_set_verify(ctx, SSL_VERIFY_NONE, NULL);

    return ctx;

}


void *worker_function(void *ptr) {

    struct event_base **evbase = (struct event_base **)ptr;
    ignore_sigpipe();
    *evbase = event_base_new();

    event_base_loop(*evbase, EVLOOP_NO_EXIT_ON_EMPTY);

    return NULL;

}

int main(int arc, char **argv) {

    SSL_CTX *ctx;
    struct evconnlistener *listener;
    struct event_base *evbase;
    struct sockaddr_in sin;

    memset(&sin, 0, sizeof(sin));
    sin.sin_family = AF_INET;
    sin.sin_port = htons(443);
    sin.sin_addr.s_addr = htonl(INADDR_ANY);

    ignore_sigpipe();
    ctx = ssl_init();

    evthread_use_pthreads();
    evbase = event_base_new();

    listener = evconnlistener_new_bind(
                        evbase, acceptcb, (void *)ctx,
                        LEV_OPT_CLOSE_ON_FREE | LEV_OPT_REUSEABLE, 1024,
                        (struct sockaddr *)&sin, sizeof(sin));

    int i;
    for (i = 0; i < NUM_THREADS; i++) {
        pthread_create(&thread[i], NULL, worker_function, (void
*)&thread_base[i]);
    }

    event_base_loop(evbase, 0);

    return 0;

}

2012/12/17 Nick Mathewson <nickm@xxxxxxxxxxxxx>:
> On Sun, Dec 16, 2012 at 10:52 AM, Björn K. <bjoernk2@xxxxxxxxxxxxxx> wrote:
>> Hello,
>>
>> I'm trying to write a simple https server which utilizes multiple cpu cores.
>> I call evthread_use_pthreads() and create several pthreads
>
> I can't tell without looking at the rest of the program, but are you
> sharing one event_base among all these threads? You can't run
> event_base_loop or event_base_dispatch on the same event_base from
> more than one thread at once.
>
> To do what you want, you could either have one event_base per thread,
> and some mechanism to assign each incoming connection gets assigned to
> a separate event_base, or you could have one event_base, and use
> worker threads to do the work of actually parsing HTTP and answering
> HTTP requests.
>
> For an example of an existing library that does what you want
> (multithreaded high-performance HTTP with SSL in Libevent), see Mark
> Ellzey's libevhtp at https://github.com/ellzey/libevhtp .
>
> yrs,
> --
> Nick
> ***********************************************************************
> To unsubscribe, send an e-mail to majordomo@xxxxxxxxxxxxx with
> unsubscribe libevent-users    in the body.
***********************************************************************
To unsubscribe, send an e-mail to majordomo@xxxxxxxxxxxxx with
unsubscribe libevent-users    in the body.