内存映射在coping gc中的应用

mmap在coping gc中应用

主要关于一个使用mmap对baker gc算法进行优化的小demo。

源码地址：pdos.csail.mit.edu/6.S081/2020/lec/baker.c

论文地址：appel-li.pdf (mit.edu)

先让大家看看baker算法对coping gc优化的精髓部分

struct elem *
new(void) {
  struct elem *n;
  
  // 加锁避免mutator thread和collector thread同时访问
  pthread_mutex_lock(&lock);
  
  // 这里是重中之重
  // 这里就是体现baker算法精髓的地方了
  if (collecting && scanned < to_free_start) {
    scan(scanned);
    if (scanned >= to_free_start) {
      end_collecting();
    }
  }
  // 空间不足
  if (to_free_start + sizeof(struct obj) >= to + SPACESZ) {
    flip();
  }
  n = (struct elem *) alloc();
  pthread_mutex_unlock(&lock);
  
  return n;
}

我们就正式开始看源码，听说这个小demo有很多不足的地方。但是笔者对gc确实不太了解，真的看不来哈哈。

#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <unistd.h>
#include <string.h>
#include <signal.h>
#include <stdint.h>
#include <sys/mman.h>

#include <assert.h>
#include <fcntl.h>
#include <pthread.h>

//
// Toy implementation of baker's GC algorithm for illustration in lecture.
//

#define LISTSZ 300

// Only one type of object: list elements
// 模拟一个对象
struct elem {
  int val;
  struct elem *next;
};

//
// API to collector
//

// 检查一个对象指针是否在to-space中
struct elem *readptr(struct elem **ptr);
// 获取一个新的对象
struct elem *new();

//
// Mutator
//

// The roots of the mutator
struct elem *root_head;
struct elem *root_last;

// 用来debug的函数
void
print_clist() {
  printf("list: %p %p\n", root_head, root_last);
  struct elem *e = root_head;
  do {
    printf("elem: %d %p %p\n", readptr(&e)->val, readptr(&e), readptr(&e)->next);
    e = readptr(&e)->next;
  } while (readptr(&e) != root_head);
  printf("\n");
}

void
check_clist(int l) {
  int sum = 0;
  struct elem *e = readptr(&root_head);

  do {
    sum += readptr(&e)->val;
    e = readptr(&e)->next;
  } while (readptr(&e) != root_head);
  assert(sum == (l * (l-1))/2);
}


// Make a circular list
void
make_clist(void) {
  struct elem *e;

  // 循环链表
  root_head = new();
  readptr(&root_head)->val = 0;
  readptr(&root_head)->next = readptr(&root_head);
  root_last = readptr(&root_head);

  // 将元素添加到对象图中
  for (int i = 1; i < LISTSZ; i++) {
    e = new();
    readptr(&e)->next = readptr(&root_head);
    readptr(&e)->val = i;
    root_head = readptr(&e);
    readptr(&root_last)->next = readptr(&root_head);
    check_clist(i+1);
  }
}

// 启动一个线程模拟现实中new程序不断产生对象的过程
void *
app_thread(void *x) {
  for (int i = 0; i < 1000; i++) {
    make_clist();
    check_clist(LISTSZ);
  }
}

//
// Collector
//
// gc的实现

#define VM 1     // Using VM

struct obj {
  struct elem e;
  struct obj *newaddr;   // if set, obj already has been copied to to-space
						 // 对象在to-space的地址
						 // 对象开始是在from中的，但是随着时间的推移
						 // 当form-space中的空间不足时，gc会将form-space中
						 // 存活的对象拷贝到to-space，并将form-space中的所有内存视为garbage
						 // 将to-space和form-space中的地址指针交换
						 // 然后就能够在新的form-space中new对象了
  struct obj *pad;       // Hack: ensure struct obj divides page size
};
  
#define PGSIZE 4096
#define SPACESZ (3*PGSIZE)    // size of space
#define TOFROM (2*SPACESZ)    // size of to and from
#define NROOT 2               // number of roots

// the space GC allocates in and copies to
static void *to; 				// to-space
static void *to_free_start;		// 表示to-space的空闲区域的起始地址
								// 也就是collector将object拷贝到这个
								// to-spcae中的地址
								//所以当mutator thread访问到从to_free_satrt到mutator + MAP_FAILED
								// 这段空间是就会产生page fault
static void *scanned;			// to-space中分为unscanned和scaned区域

// the space GC is collecting
static void *from;

static int collecting;         // Collecting?

// 在mutator thread和collector thread中address space是同一块空间
// 这是论文中提到的map2语意
// 这里是将mutator线程中的unscaned部分映射为只读
// 将collector线程中的unscaned部分映射为读写部分
// 所以，在mutator线程访问到这一块地址时，就会触发page fault
// 跟我们在内核中看到的page fault是一样的，
// 这里就用到了论文中提到的trap语意，具体是通过sigaction来实现
// sigaction在colletor线程中注册了一个信号处理函数，
// 当mutator线程访问时触发这个page fault，内核就会产生一个信号
// 并被colletor线程捕获，这时，collector线程就会马上处理这一块产生
// page fault的页：将这个页对应从from-space中拷贝过来，
// 并恢复这个页的权限，这里体现了论文中的unport语意，具体体现为mprotection()

// 我们还需要注意的是，
// 两个线程都是在同一个进程中的，这里是将同一个进程中的同一个物理块
// 映射到同一个进程中不同虚拟地址中，因为线程之间是共享进程资源的
// 线程只有自己的线程栈是私有的
// Address where from+to space is mapped for mutator
// 映射到mutator线程的地址空间
static void *mutator = NULL;
// Address where from+to space is mapped for collector
// gc的地址空间
static void *collector = NULL;

static pthread_mutex_t lock;
static pthread_cond_t cond;
static int done;

#define APPADDR(a) (assert((void *)(a) >= mutator && (void *) (a) < mutator + TOFROM))
#define GCADDR(a) (assert((void *)(a) >= collector && (void *)(a) < collector + TOFROM))

// Convert from collector address to app address
// 基址地址 + 偏移量
void *
appaddr(void *a) {
  assert(a >= collector && a < collector + TOFROM);
  return (a - collector) + mutator;
}

// Convert from app address to gc address
void *
gcaddr(void *a) {
  assert(a >= mutator && a < mutator + TOFROM);
  return (a - mutator) + collector;
}

// 在to-space上申请一块内存放置新对象
struct obj *
alloc() {
  struct obj *o = (struct obj *) to_free_start;
  assert(to_free_start + sizeof(struct obj) < to + SPACESZ);
  to_free_start += sizeof(struct obj);
  return o;
}

struct obj *
copy(struct obj *o) {
  struct obj *n = gcaddr(alloc());
  n->e = o->e;
  o->newaddr = n;
  n->newaddr = NULL;
  return (appaddr(n));
}

int
in_from(struct obj *o) {
  return ((void *) o >= from && ((void *) o < from+SPACESZ));
}

// Forward pointer to to-space. If pointer is in from-space, make a
// copy of the object in to-space and return pointer to copy. If
// object was already copied (there may several pointers to the same
// object), return the pointer to the to-space object.  If pointer is
// already in to-space, just return the pointer.
// 将指针发送到to-space
// 如果指针在from-space，将这个对象拷贝到to-space并返回这个拷贝的对象
// 如果这个对象已经被拷贝了（这个对象有一个指针指向to-space中的副本）
//		返回在to-space中的对象
// 如果指针已经在to-space，直接返回这个指针
struct obj *
forward(struct obj *o) {
  struct obj *n = o;
  if (in_from(o)) {
    if (o->newaddr == 0) {   // not copied yet?
      n = copy(o);
      printf("forward: copy %p (%d) to %p\n", o, o->e.val, n);
    } else {
      n = o->newaddr;
      printf("forward: already copied %p (%d) to %p\n", o, o->e.val, n);
    }
  }
  return n;
}

// Scan a page of unscanned objects and forward the pointers in the
// objects.
// 扫面所未被扫描的对象并将这些指针发送到
int
scan(void *start) {
  int sz = 0;
  void *next = gcaddr(start);
  printf("scan %p unscanned %p\n", gcaddr(start), gcaddr(to_free_start));
  while (next < gcaddr(start)+PGSIZE && next < gcaddr(to_free_start)) {
    struct obj *o = (struct obj *) next;
    if(o->e.next != 0)
      printf("scan: %p %d %p\n", o, o->e.val, o->e.next);
	// 将from-space中的old object指向对应在to-space中的新对象
    o->e.next = (struct elem *) forward((struct obj *) (o->e.next));
    next += sizeof(struct obj);
  }
  scanned = appaddr(next);
  printf("scan done %p\n", to_free_start);
}

void
end_collecting() {
  printf("collection done\n");
  memset(from, 0, SPACESZ);
  collecting = 0;
}

// 将to-spcae和from-space中指针进行交换
void
flip() {
  char *tmp = to;

  printf("flip spaces\n");

  assert(!collecting);
  to = from;
  to_free_start = from;
  from = tmp;
	
  // 开始清
  collecting = 1;
  scanned = to;

#ifdef VM
// 将to-space映射为PROT_NONE
// 当mutator访问到这个块空间的时候引发page fault
  if (mprotect(to, SPACESZ, PROT_NONE) < 0) {
    fprintf(stderr, "Couldn't unmap to space; %s\n", strerror(errno));
    exit(EXIT_FAILURE);
  }
#endif

  // move root_head and root_last to to-space
  root_head = (struct elem *) forward((struct obj *)root_head);
  root_last = (struct elem *) forward((struct obj *)root_last);

  pthread_cond_broadcast(&cond);
}

// Without VM: check if pointer should be forwarded.  With VM, do
// nothing; the segfault handler will fix things up and when it
// returns the pointer will be pointing to object in to-space.
// 如果没有使用虚拟内存机制，就需要检查指针是否被发送到to-space，
// 但是如果使用了虚拟内存机制，我们只需要捕获产生的page fault，然后让
// collector thread去处理即可
struct elem *
__attribute__ ((noinline))
readptr(struct elem **p) {
#ifndef VM
  struct obj *o = forward((struct obj *) (*p));
  *p = (struct elem *) o;
#endif
  return *p;
}

struct elem *
new(void) {
  struct elem *n;
  
  // 加锁避免mutator thread和collector thread同时访问
  pthread_mutex_lock(&lock);
  if (collecting && scanned < to_free_start) {
    scan(scanned);
    if (scanned >= to_free_start) {
      end_collecting();
    }
  }
  // 空间不足
  if (to_free_start + sizeof(struct obj) >= to + SPACESZ) {
    flip();
  }
  n = (struct elem *) alloc();
  pthread_mutex_unlock(&lock);
  
  return n;
}

void *
collector_thread(void *x) {
  printf("start collector\n");
  while (!done) {
    pthread_mutex_lock(&lock);
    if (collecting && scanned < to_free_start) {
      printf("collector: scan\n");
      scan(scanned);
      if (scanned >= to_free_start) {
        end_collecting();
      }
    }
	// 如果并不需要进行垃圾回收
    while (!collecting) {
      pthread_cond_wait(&cond, &lock);
    }
    pthread_mutex_unlock(&lock);
  }
  printf("done collector\n");
}

// Rounds v down to an alignment a
#define align_down(v, a) ((v) & ~((typeof(v))(a) - 1))
// 信号处理函数，在collector线程中执行的函数
// sig -> 信号码
// ctx 上下文
// 这是信号处理函数的标配格式
// sighandler_t signal(int signum, sighandler_t handler);
static void
handle_sigsegv(int sig, siginfo_t *si, void *ctx)
{
  uintptr_t fault_addr = (uintptr_t)si->si_addr;
  double *page_base = (double *)align_down(fault_addr, PGSIZE);

  printf("fault at adddr %p (page %p)\n", fault_addr, page_base);

  pthread_mutex_lock(&lock);
  scan(page_base);
  pthread_mutex_unlock(&lock);
  
  if (mprotect(page_base, PGSIZE, PROT_READ|PROT_WRITE) < 0) {
    fprintf(stderr, "Couldn't mprotect to-space page; %s\n", strerror(errno));
    exit(EXIT_FAILURE);
  }
}

//
// Initialization
//

static void
setup_spaces(void)
{
  struct sigaction act;
  int shm;

  // make a shm obj for to-space and from-space
  // shm_open 创建一个虚拟内存对象
  if ((shm = shm_open("baker", O_CREAT|O_RDWR|O_TRUNC, S_IRWXU)) < 0) {
    fprintf(stderr, "Couldn't shm_open: %s\n", strerror(errno));
    exit(EXIT_FAILURE);
  }

  // 这是我从man手册上读到的，我想这就是原因
  //   After a suc‐
  //     cessful shm_unlink(), attempts to shm_open() an object with
  //     the  same name fail (unless O_CREAT was specified, in which
  //     case a new, distinct object is created).
  if (shm_unlink("baker") != 0) {
    fprintf(stderr, "shm_unlink failed: %s\n", strerror(errno));
    exit(EXIT_FAILURE);
  }
  
  // 增长这个虚拟内存对象的大小，这个虚拟内次年是由系统分配的
  // 如果shm.size() > TOFROM 减小空间
  // 否则增长空间
  if (ftruncate(shm, TOFROM) != 0) {
    fprintf(stderr, "ftruncate failed: %s\n", strerror(errno));
    exit(EXIT_FAILURE);
  }

  // map it once for mutator
  mutator = mmap(NULL, TOFROM, PROT_READ|PROT_WRITE, MAP_SHARED, shm, 0);
  if (mutator == MAP_FAILED) {
    fprintf(stderr, "Couldn't mmap() from space; %s\n", strerror(errno));
    exit(EXIT_FAILURE);
  }

  // map it once for collector
  collector = mmap(NULL, TOFROM, PROT_READ|PROT_WRITE, MAP_SHARED, shm, 0);
  if (collector == MAP_FAILED) {
    fprintf(stderr, "Couldn't mmap() from space; %s\n", strerror(errno));
    exit(EXIT_FAILURE);
  }

  from = mutator;
  to = mutator + SPACESZ;
  to_free_start = to;

  // Register a signal handler to capture SIGSEGV.
  act.sa_sigaction = handle_sigsegv;
  act.sa_flags = SA_SIGINFO;
  sigemptyset(&act.sa_mask);
  // 注册一个信号处理函数
  if (sigaction(SIGSEGV, &act, NULL) == -1) {
    fprintf(stderr, "Couldn't set up SIGSEGV handler: %s\n", strerror(errno));
    exit(EXIT_FAILURE);
  }
}


int
main(int argc, char *argv[])
{
  pthread_t the_app;
  pthread_t the_collector;
  
  printf("sizeof %d %d\n", sizeof(struct elem), sizeof(struct obj));
  assert(PGSIZE % sizeof(struct obj) == 0);
  setup_spaces();

  pthread_mutex_init(&lock, NULL);
  pthread_cond_init(&cond, NULL);

  if (pthread_create(&the_app, NULL, app_thread,  NULL) != 0) {
    fprintf(stderr, "Couldn't start app thread: %s\n", strerror(errno));
    exit(EXIT_FAILURE);
  }

// 如果不引入虚拟内存机制
// 那么collector和mutator只能够交替执行、
// 并不能正真的并发

#ifdef VM
  if (pthread_create(&the_collector, NULL, collector_thread,  NULL) != 0) {
    fprintf(stderr, "Couldn't start app thread: %s\n", strerror(errno));
    exit(EXIT_FAILURE);
  }
#endif

  if (pthread_join(the_app, NULL) != 0) {
    fprintf(stderr, "Couldn't join app thread: %s\n", strerror(errno));
    exit(EXIT_FAILURE);
  }
  
  done = 1;
  
  return 0;
}


// mmap还是比较有趣的，哈哈
// 总结一下：
// 通过使用vm，我们能够使原先不能并行执行（强制加大锁我也没办法）的
// baker gc能够并行地执行了
// baker的主要思想是均摊，这种思想在平时设计系统的时候也能够用上
// 刚好我刚刚写的一个数据库的aof重写模块就用上
// 哈哈哈哈