Memcached源码拆分：Libevent_Thread

http://www.cnblogs.com/aga-j/archive/2011/11/04/2230603.html

WorkerThreads.cpp

  1 /*
  2 #include"libevent_thread.h"
  3 #include<event.h>
  4 #include<pthread.h>
  5 #include"conn.h"
  6 #include"cq.h"
  7 #include<unistd.h>
  8 #include<string.h>
  9 */
 10 #include"WorkerThreads.h"
 11 #include<stdio.h>
 12 
 13 /*
 14     WorkerThreads::WorkerThreads(int threadCount=1):nthreads(threadCount),threads(NULL)
 15     {
 16         init_count=0;
 17         pthread_mutex_init(&init_lock,NULL);
 18         pthread_cond_init(&init_cond,NULL);
 19     }
 20 */
 21 
 22     void WorkerThreads::initiate()
 23     {
 24         int i;
 25 
 26         threads=(LIBEVENT_THREAD*)calloc(nthreads,sizeof(LIBEVENT_THREAD));        //LIBEVENT_THREAD,加入libevent元素的thread结构 “数组”
 27         if(!threads)
 28         {
 29             perror("can't allocate thread des");
 30             exit(1);
 31         }
 32 
 33         for(i=0;i<nthreads;i++)                                                    //设置thread和thread中的libevent所需属性
 34         {
 35             int fds[2];
 36             if(pipe(fds))                                                        //thread和主线程的通信pipe
 37             {
 38                 perror("can't create notify pipe");
 39                 exit(1);
 40             }
 41 
 42             threads[i].notify_receive_fd=fds[0];
 43             threads[i].notify_send_fd=fds[1];
 44 
 45             setup_event_thread(&threads[i]);                                    //设置thread和thread中的libevent所需属性
 46         }
 47 
 48         for(i=0;i<nthreads;i++)
 49         {
 50             create_worker(worker_libevent,&threads[i]);                            //启动thread
 51         }
 52 
 53         pthread_mutex_lock(&init_lock);
 54         while( init_count < nthreads)
 55         {
 56             pthread_cond_wait(&init_cond,&init_lock);
 57         }
 58         pthread_mutex_unlock(&init_lock);
 59 
 60         printf("finish\n");
 61     }
 62 
 63     void WorkerThreads::setup_event_thread(LIBEVENT_THREAD *me)
 64     {
 65         me->base=event_init();//every thread has its own event_base
 66         
 67 //2.0 has event_config
 68         /*
 69         struct event_config *cfg=event_config_new();
 70         event_config_avoid_method(cfg,"epoll");
 71         me->base=event_base_new_with_config(cfg);
 72         event_config_free(cfg);
 73 */
 74         //in order to use libevent on file,use this method
 75 
 76         if(!me->base)
 77         {
 78             fprintf(stderr,"can't allocate event base\n");
 79             exit(1);
 80         }
 81 
 82         event_set(&me->notify_event,me->notify_receive_fd,                        //设置 监听事件 和 处理函数
 83             EV_READ|EV_PERSIST, thread_libevent_process,me);    
 84         event_base_set(me->base,&me->notify_event);
 85 
 86         if(event_add(&me->notify_event,0)==-1)
 87         {
 88             fprintf(stderr,"can't monitor libevent notify pipe\n");
 89             exit(1);
 90         }
 91 
 92         //why initiate conn_queue here?
 93         me->new_conn_queue=(conn_queue*)malloc(sizeof(struct conn_queue));        //内部的conn_queue
 94         if(me->new_conn_queue==NULL)
 95         {
 96             perror("Failed to allocate memory for connection queue");
 97             exit(EXIT_FAILURE);
 98         }
 99 
100         cq_init(me->new_conn_queue);
101     }
102     
103     void thread_libevent_process(int fd,short which,void *arg)                    //处理函数，即当主线程通知workerThread时，主线程会插入一个item到某个thread的queue中，queue是一个工具类
104 //workerThread将item pop出来并封装为conn，封装期间就建立了和item所指向的对象的联系，也使用libevent完成
105     {
106         LIBEVENT_THREAD *me=(LIBEVENT_THREAD*)arg;
107         CQ_ITEM *item;
108 
109         char buf[1];
110 
111         if(read(fd,buf,1)!=1)
112             fprintf(stderr,"can't read from libevent pipe\n");
113 
114         item=cq_pop(me->new_conn_queue);
115         printf("item fd is:%d\n",item->sfd);
116         if(NULL!=item)
117         {
118             conn *c= conn_new (item->sfd,item->init_state,item->event_flags,
119                 item->read_buffer_size,item->transport,me->base);
120     
121             if(NULL==c)
122             {
123                 if( IS_UDP(item->transport))
124                 {
125                     fprintf(stderr,"can't listen for events on UDP\n");
126                     exit(1);
127                 }
128                 else
129                 {
130                     fprintf(stderr,"can't listen for events on fd %d\n",item->sfd);
131                     close(item->sfd);
132                 }
133             }
134             else
135             {
136                 c->thread=me;
137                 //test
138                 write(item->read_buffer_size,"a",1);
139             }
140             cqi_free(item);
141         }
142 
143     }
144     
145     void WorkerThreads::create_worker(void *(*func)(void*),void *arg)
146     {
147         pthread_t thread;
148         pthread_attr_t attr;
149         int ret;
150 
151         pthread_attr_init(&attr);
152 
153         if((ret=pthread_create(&thread,&attr,func,arg))!=0)                
154         {
155             fprintf(stderr,"can't create thread: %s\n",strerror(ret));
156             exit(1);
157         }
158     }
159     
160     void* worker_libevent(void *arg)
161     {
162         LIBEVENT_THREAD *me = (LIBEVENT_THREAD*)arg;
163         pthread_mutex_lock(&init_lock);
164         init_count++;
165         pthread_cond_signal(&init_cond);
166         pthread_mutex_unlock(&init_lock);
167 
168         event_base_loop(me->base,0);
169         return NULL;
170     }
171 
172     
173 
174     

conn.cpp

  1 //#include<pthread.h>
  2 //#include<event.h>
  3 #include"conn.h"
  4 
  5 #include<unistd.h>
  6 
  7 static pthread_mutex_t conn_lock=PTHREAD_MUTEX_INITIALIZER;
  8 static int freecurr;
  9 static int freetotal;
 10 
 11 
 12 
 13 static conn **freeconns;
 14 
 15 //static                    //why can not use static 
 16 void conn_init(void) {                                                        //每个Worker的
 17     freetotal = 200;
 18     freecurr = 0;
 19     if ((freeconns =(conn**)calloc(freetotal, sizeof(conn *))) == NULL) {
 20         fprintf(stderr, "Failed to allocate connection structures\n");
 21     }
 22     return;
 23 }
 24 
 25 conn *conn_new(const int sfd,enum conn_states init_state,const int event_flags,
 26                const int read_buffer_size,enum network_transport transport,
 27 struct event_base *base)
 28 {
 29     conn *c=conn_from_freelist();
 30     if(NULL==c)
 31     {
 32         if(!(c=(conn*)calloc(1,sizeof(conn))))
 33         {
 34             fprintf(stderr,"calloc()\n");
 35             return NULL;
 36         }
 37         //initiate c
 38 /*
 39 {
 40     MEMCACHED_CONN_CREATE(c);
 41 
 42         c->rbuf = c->wbuf = 0;
 43         c->ilist = 0;
 44         c->suffixlist = 0;
 45         c->iov = 0;
 46         c->msglist = 0;
 47         c->hdrbuf = 0;
 48 
 49         c->rsize = read_buffer_size;
 50         c->wsize = DATA_BUFFER_SIZE;
 51         c->isize = ITEM_LIST_INITIAL;
 52         c->suffixsize = SUFFIX_LIST_INITIAL;
 53         c->iovsize = IOV_LIST_INITIAL;
 54         c->msgsize = MSG_LIST_INITIAL;
 55         c->hdrsize = 0;
 56 
 57         c->rbuf = (char *)malloc((size_t)c->rsize);
 58         c->wbuf = (char *)malloc((size_t)c->wsize);
 59         c->ilist = (item **)malloc(sizeof(item *) * c->isize);
 60         c->suffixlist = (char **)malloc(sizeof(char *) * c->suffixsize);
 61         c->iov = (struct iovec *)malloc(sizeof(struct iovec) * c->iovsize);
 62         c->msglist = (struct msghdr *)malloc(sizeof(struct msghdr) * c->msgsize);
 63 
 64         //根据配置大小来分配，可能会分配失败
 65         if (c->rbuf == 0 || c->wbuf == 0 || c->ilist == 0 || c->iov == 0 ||
 66                 c->msglist == 0 || c->suffixlist == 0) {
 67             conn_free(c);
 68             fprintf(stderr, "malloc()\n");
 69             return NULL;
 70         }
 71         //end initiate c
 72 }    
 73 */
 74     }
 75     //initiate c
 76 /*
 77 {
 78     c->transport = transport;                 //传输方式
 79     c->protocol = settings.binding_protocol;  //传输协议
 80 
 81     // unix socket mode doesn't need this, so zeroed out.  but why
 82     //  is this done for every command?  presumably for UDP
 83     //  mode.  
 84     if (!settings.socketpath) {
 85         c->request_addr_size = sizeof(c->request_addr);
 86     } else {
 87         c->request_addr_size = 0;
 88     }
 89 
 90     if (settings.verbose > 1) {
 91         if (init_state == conn_listening) {
 92             fprintf(stderr, "<%d server listening (%s)\n", sfd,
 93                 prot_text(c->protocol));
 94         } else if (IS_UDP(transport)) {
 95             fprintf(stderr, "<%d server listening (udp)\n", sfd);
 96         } else if (c->protocol == negotiating_prot) {
 97             fprintf(stderr, "<%d new auto-negotiating client connection\n",
 98                     sfd);
 99         } else if (c->protocol == ascii_prot) {
100             fprintf(stderr, "<%d new ascii client connection.\n", sfd);
101         } else if (c->protocol == binary_prot) {
102             fprintf(stderr, "<%d new binary client connection.\n", sfd);
103         } else {
104             fprintf(stderr, "<%d new unknown (%d) client connection\n",
105                 sfd, c->protocol);
106             assert(false);
107         }
108     }
109 
110 //信息赋值给conn结构，conn的构造把item全用上了
111     c->sfd = sfd;
112     c->state = init_state;
113     c->rlbytes = 0;
114     c->cmd = -1;
115     c->rbytes = c->wbytes = 0;
116     c->wcurr = c->wbuf;
117     c->rcurr = c->rbuf;
118     c->ritem = 0;
119     c->icurr = c->ilist;
120     c->suffixcurr = c->suffixlist;
121     c->ileft = 0;
122     c->suffixleft = 0;
123     c->iovused = 0;
124     c->msgcurr = 0;
125     c->msgused = 0;
126 
127     c->write_and_go = init_state;
128     c->write_and_free = 0;
129     c->item = 0;
130 
131     c->noreply = false;
132 }
133 */
134     //end initiate c
135     
136     c->sfd=sfd;
137     printf("test fd is:%d\n",sfd);
138     event_set(&c->event,sfd,event_flags,event_handler,(void*)c);
139     event_base_set(base,&c->event);
140     c->ev_flags=event_flags;
141 
142     if(event_add(&c->event,0)==-1)
143     {
144         if( conn_add_to_freelist(c))
145             conn_free(c);
146         perror("event_add");
147         return NULL;
148     }
149 
150     //MEMCACHED_CONN_ALLOCATE(c->sfd);
151     
152     return c;
153 }
154 
155 conn *conn_from_freelist()
156 {
157     conn *c;
158     pthread_mutex_lock(&conn_lock);
159 
160     if(freecurr>0)
161         c=freeconns[--freecurr];
162     else
163         c=NULL;
164 
165     pthread_mutex_unlock(&conn_lock);
166 }
167 
168 void event_handler(const int fd,const short which,void *arg)
169 {
170     //real handler
171     char buf[128];
172     int rc=read(fd,buf,sizeof(buf));
173     buf[rc]='\0';
174     printf("%s\n",buf);
175 }
176 
177 bool conn_add_to_freelist(conn *c) 
178 {
179     bool ret = true;
180     pthread_mutex_lock(&conn_lock);
181 
182     if (freecurr < freetotal) 
183     {
184         freeconns[freecurr++] = c;
185         ret = false;
186     }
187     else 
188     {
189         // try to enlarge free connections array 
190         size_t newsize = freetotal * 2;
191         conn **new_freeconns = (conn**)realloc(freeconns, sizeof(conn *) * newsize);
192         if (new_freeconns) 
193         {                
194             freetotal = newsize;
195             freeconns = new_freeconns;
196             freeconns[freecurr++] = c;        
197             ret = false;
198         }
199     }
200 
201     pthread_mutex_unlock(&conn_lock);
202     return ret;
203 }
204 
205 void conn_free(conn *c) {
206     if (c) 
207     {
208         /*
209         MEMCACHED_CONN_DESTROY(c);
210         if (c->hdrbuf)              
211         free(c->hdrbuf);
212         if (c->msglist)
213         free(c->msglist);        
214         if (c->rbuf)        
215         free(c->rbuf);        
216         if (c->wbuf)        
217         free(c->wbuf);            
218         if (c->ilist)        
219         free(c->ilist);        
220         if (c->suffixlist)        
221         free(c->suffixlist);        
222         if (c->iov)
223         free(c->iov);    
224 */
225         free(c);    
226     }
227 }