Couchstore部分是couchbase中负责具体实施文档存取和建立索引的组件，相关的硬盘操作和缓冲操作都在这里实现。这一部分源码有个包含在源码内的总体测试程序testapp.c，其中包括了对重点函数的测试，在此就以这个测试程序为入口分析重点函数的机制。
int main(int argc, const char *argv[])
{
int doc_counts[] = { 4, 69, 666, 9090 };
unsigned i;
const char *small_doc_tpl = "{\"test_doc_index\":%d}";
const char *large_doc_tpl =
"{"
"\"test_doc_index\":%d,"
"\"field1\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\","
"\"field2\": \"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\","
"\"field3\": \"cccccccccccccccccccccccccccccccccccccccccccccccccc"
"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\""
"}";
if (argc > 1)
strcpy(testfilepath, argv[1]);
printf("Using test database at %s\n", testfilepath);
test_bitfield_fns();
test_open_file_error();
fprintf(stderr, "OK \n");
test_dump_empty_db();
fprintf(stderr, " OK\n");
test_save_doc();
fprintf(stderr, " OK\n");
for (i = 0; i < (sizeof(doc_counts) / sizeof(int)); ++i) {
test_save_docs(doc_counts[i], small_doc_tpl);
fprintf(stderr, " OK\n");
test_save_docs(doc_counts[i], large_doc_tpl);
fprintf(stderr, " OK\n");
}
test_local_docs();
fprintf(stderr, " OK\n");
test_compressed_doc_body();
fprintf(stderr, " OK\n");
test_changes_no_dups();
fprintf(stderr, " OK\n");
mb5086();
fprintf(stderr, " OK\n");
unlink(testfilepath);
test_huge_revseq();
fprintf(stderr, " OK\n");
unlink(testfilepath);
TestCollateJSON();
TestCouchIndexer();
// make sure os.c didn't accidentally call close(0):
assert(lseek(0, 0, SEEK_CUR) >= 0 || errno != EBADF);
return 0;
}
测试程序并不复杂，test_*形式的函数显然就是测试对应功能的函数。接下来就顺序分析每一个test函数。但是因为不同测试的复杂程度不一样，所以比较复杂的test会在另外的部分专题分析，如果完成了就在这里补上链接，未完成就直接空着了。
先看第一个测试函数：
static void test_bitfield_fns(void)
{
assert(sizeof(cs_off_t) == 8);
assert(sizeof(raw_08) == 1);
assert(sizeof(raw_16) == 2);
assert(sizeof(raw_32) == 4);
assert(sizeof(raw_40) == 5);
assert(sizeof(raw_48) == 6);
struct {
raw_08 a;
raw_48 b;
raw_16 c;
raw_40 d;
raw_32 e;
raw_08 f;
} packed;
assert(sizeof(packed) == 19);
raw_kv_length kv;
assert(sizeof(kv) == 5);
kv = encode_kv_length(1234, 123456);
uint32_t klen, vlen;
decode_kv_length(&kv, &klen, &vlen);
assert(klen == 1234);
assert(vlen == 123456);
test_raw_08(0);
test_raw_08(UINT8_MAX);
test_raw_16(0);
test_raw_16(12345);
test_raw_16(UINT16_MAX);
test_raw_32(0);
test_raw_32(12345678);
test_raw_32(UINT32_MAX);
uint8_t expected1[8] = {0x12, 0x34, 0x56, 0x78, 0x90};
test_raw_40(0x1234567890LL, expected1);
uint8_t expected2[8] = {0x09, 0x87, 0x65, 0x43, 0x21};
test_raw_40(0x0987654321LL, expected2);
uint8_t expected3[8] = {0x12, 0x34, 0x56, 0x78, 0x90, 0xAB};
test_raw_48(0x1234567890ABLL, expected3);
uint8_t expected4[8] = {0xBA, 0x98, 0x76, 0x54, 0x32, 0x10};
test_raw_48(0xBA9876543210LL, expected4);
}第二个测试函数：
static void test_open_file_error(void)
{
fprintf(stderr, "opening nonexistent file errors... ");
fflush(stderr);
unlink(testfilepath);
Db *db;
int errcode = couchstore_open_db(testfilepath, 0, &db);
if(errcode != 0) {
print_os_err();
}
assert(errcode == COUCHSTORE_ERROR_NO_SUCH_FILE);
// make sure os.c didn't accidentally call close(0):
assert(lseek(0, 0, SEEK_CUR) >= 0 || errno != EBADF);
}第三个测试函数：
static void test_dump_empty_db(void)
{
fprintf(stderr, "dump empty db... ");
fflush(stderr);
unlink(testfilepath);
Db *db;
couchstore_error_t errcode;
try(couchstore_open_db(testfilepath, COUCHSTORE_OPEN_FLAG_CREATE, &db));
try(couchstore_close_db(db));
try(couchstore_open_db(testfilepath, 0, &db));
dump_count(db);
assert(counters.totaldocs == 0);
assert(counters.deleted == 0);
DbInfo info;
assert(couchstore_db_info(db, &info) == COUCHSTORE_SUCCESS);
assert(strcmp(info.filename, testfilepath) == 0);
assert(info.last_sequence == 0);
assert(info.doc_count == 0);
assert(info.deleted_count == 0);
assert(info.space_used == 0);
assert(info.header_position == 0);
couchstore_close_db(db);
cleanup:
assert(errcode == 0);
}第四个测试函数：
static void test_save_doc(void)
{
fprintf(stderr, "save_doc... ");
fflush(stderr);
int errcode = 0;
docset_init(4);
SETDOC(0, "doc1", "{\"test_doc_index\":1}", zerometa);
SETDOC(1, "doc2", "{\"test_doc_index\":2}", zerometa);
SETDOC(2, "doc3", "{\"test_doc_index\":3}", zerometa);
SETDOC(3, "doc4", "{\"test_doc_index\":4}", zerometa);
unlink(testfilepath);
Db *db;
try(couchstore_open_db(testfilepath, COUCHSTORE_OPEN_FLAG_CREATE, &db));
try(couchstore_save_document(db, &testdocset.docs[0],
&testdocset.infos[0], 0));
try(couchstore_save_document(db, &testdocset.docs[1],
&testdocset.infos[1], 0));
try(couchstore_save_document(db, &testdocset.docs[2],
&testdocset.infos[2], 0));
try(couchstore_save_document(db, &testdocset.docs[3],
&testdocset.infos[3], 0));
try(couchstore_commit(db));
couchstore_close_db(db);
// Check that sequence numbers got filled in
unsigned i;
for (i = 0; i < 4; ++i) {
assert(testdocset.infos[i].db_seq == i + 1);
}
//Read back
try(couchstore_open_db(testfilepath, 0, &db));
try(couchstore_changes_since(db, 0, 0, docset_check, &testdocset));
assert(testdocset.counters.totaldocs == 4);
assert(testdocset.counters.deleted == 0);
DbInfo info;
assert(couchstore_db_info(db, &info) == COUCHSTORE_SUCCESS);
assert(info.last_sequence == 4);
assert(info.doc_count == 4);
assert(info.deleted_count == 0);
assert(info.header_position == 4096);
couchstore_close_db(db);
cleanup:
assert(errcode == 0);
}
couchstore_error_t couchstore_save_document(Db *db, const Doc *doc,
DocInfo *info, couchstore_save_options options)
{
return couchstore_save_documents(db, (Doc**)&doc, (DocInfo**)&info, 1, options);
}
等到后面看到里面的函数时再具体分析。
第五个测试函数：
static void test_save_docs(int count, const char *doc_tpl)
{
int errcode = 0;
int i;
char *idBuf, *valueBuf;
Doc **docptrs;
DocInfo **nfoptrs;
sized_buf *ids;
uint64_t idtreesize = 0;
uint64_t seqtreesize = 0;
uint64_t docssize = 0;
uint64_t dbfilesize = 0;
uint64_t *sequences = NULL;
fprintf(stderr, "save_docs (doc count %d)... ", count);
fflush(stderr);
docset_init(count);
srandom(0xdeadbeef); // doc IDs should be consistent across runs
for (i = 0; i < count; ++i) {
idBuf = (char *) malloc(sizeof(char) * 32);
assert(idBuf != NULL);
int idsize = sprintf(idBuf, "doc%d-%lu", i, (unsigned long)random());
valueBuf = (char *) malloc(sizeof(char) * (strlen(doc_tpl) + 20));
assert(valueBuf != NULL);
int valsize = sprintf(valueBuf, doc_tpl, i + 1);
setdoc(&testdocset.docs[i], &testdocset.infos[i],
idBuf, idsize, valueBuf, valsize, zerometa, sizeof(zerometa));
testdocset.datasize += valsize;
}
docptrs = (Doc **) malloc(sizeof(Doc*) * count);
assert(docptrs != NULL);
for (i = 0; i < count; ++i) {
docptrs[i] = &testdocset.docs[i];
}
nfoptrs = (DocInfo **) malloc(sizeof(DocInfo*) * count);
assert(nfoptrs != NULL);
for (i = 0; i < count; ++i) {
nfoptrs[i] = &testdocset.infos[i];
}
unlink(testfilepath);
Db *db;
try(couchstore_open_db(testfilepath, COUCHSTORE_OPEN_FLAG_CREATE, &db));
assert(strcmp(couchstore_get_db_filename(db), testfilepath) == 0);
try(couchstore_save_documents(db, docptrs, nfoptrs, count, 0));
try(couchstore_commit(db));
couchstore_close_db(db);
try(couchstore_open_db(testfilepath, 0, &db));
// Read back by doc ID:
fprintf(stderr, "get by ID... ");
testdocset.pos = 0;
for (i = 0; i < count; ++i) {
DocInfo* out_info;
try(couchstore_docinfo_by_id(db, testdocset.docs[i].id.buf, testdocset.docs[i].id.size,
&out_info));
docset_check(db, out_info, &testdocset);
couchstore_free_docinfo(out_info);
}
// Read back in bulk by doc ID:
fprintf(stderr, "bulk IDs... ");
ids = malloc(count * sizeof(sized_buf));
for (i = 0; i < count; ++i) {
ids[i] = docptrs[i]->id;
}
ZERO(testdocset.counters);
try(couchstore_docinfos_by_id(db, ids, count, 0, dociter_check, &testdocset));
assert(testdocset.counters.totaldocs == count);
assert(testdocset.counters.deleted == 0);
// Read back by sequence:
fprintf(stderr, "get by sequence... ");
sequences = malloc(count * sizeof(*sequences));
testdocset.pos = 0;
for (i = 0; i < count; ++i) {
DocInfo* out_info;
sequences[i] = testdocset.infos[i].db_seq;
assert(sequences[i] == (uint64_t)i + 1);
try(couchstore_docinfo_by_sequence(db, testdocset.infos[i].db_seq, &out_info));
docset_check(db, out_info, &testdocset);
couchstore_free_docinfo(out_info);
}
// Read back in bulk by sequence:
fprintf(stderr, "bulk sequences... ");
testdocset.pos = 0;
ZERO(testdocset.counters);
try(couchstore_docinfos_by_sequence(db, sequences, count, 0, docset_check, &testdocset));
assert(testdocset.counters.totaldocs == count);
assert(testdocset.counters.deleted == 0);
// Read back using changes_since:
fprintf(stderr, "changes_since... ");
testdocset.pos = 0;
ZERO(testdocset.counters);
try(couchstore_changes_since(db, 0, 0, docset_check, &testdocset));
assert(testdocset.counters.totaldocs == count);
assert(testdocset.counters.deleted == 0);
idtreesize = db->header.by_id_root->subtreesize;
seqtreesize = db->header.by_seq_root->subtreesize;
const raw_by_id_reduce *reduce = (const raw_by_id_reduce*)db->header.by_id_root->reduce_value.buf;
docssize = decode_raw48(reduce->size);
dbfilesize = db->file.pos;
assert(dbfilesize > 0);
assert(idtreesize > 0);
assert(seqtreesize > 0);
assert(docssize > 0);
assert(idtreesize < dbfilesize);
assert(seqtreesize < dbfilesize);
assert(docssize < dbfilesize);
assert(db->header.local_docs_root == NULL);
assert((idtreesize + seqtreesize + docssize) < dbfilesize);
couchstore_close_db(db);
cleanup:
free(ids);
free(sequences);
for (i = 0; i < count; ++i) {
free(docptrs[i]->id.buf);
free(docptrs[i]->data.buf);
}
free(docptrs);
free(nfoptrs);
assert(errcode == 0);
}第六个测试函数：
static void test_local_docs(void)
{
fprintf(stderr, "local docs... ");
fflush(stderr);
int errcode = 0;
Db *db;
LocalDoc lDocWrite;
LocalDoc *lDocRead = NULL;
unlink(testfilepath);
try(couchstore_open_db(testfilepath, COUCHSTORE_OPEN_FLAG_CREATE, &db));
lDocWrite.id.buf = "_local/testlocal";
lDocWrite.id.size = 16;
lDocWrite.json.buf = "{\"test\":true}";
lDocWrite.json.size = 13;
lDocWrite.deleted = 0;
couchstore_save_local_document(db, &lDocWrite);
couchstore_commit(db);
couchstore_close_db(db);
couchstore_open_db(testfilepath, 0, &db);
couchstore_open_local_document(db, "_local/testlocal", 16, &lDocRead);
assert(lDocRead);
assert(lDocRead->json.size == 13);
assert(memcmp(lDocRead->json.buf, "{\"test\":true}", 13) == 0);
couchstore_free_local_document(lDocRead);
couchstore_close_db(db);
cleanup:
assert(errcode == 0);
}第七个测试函数：
static void test_compressed_doc_body(void)
{
fprintf(stderr, "compressed bodies... ");
fflush(stderr);
int errcode = 0;
docset_init(2);
SETDOC(0, "doc1", "{\"test_doc_index\":1, \"val\":\"blah blah blah blah blah blah\"}", zerometa);
SETDOC(1, "doc2", "{\"test_doc_index\":2, \"val\":\"blah blah blah blah blah blah\"}", zerometa);
Doc *docptrs [2] = { &testdocset.docs[0],
&testdocset.docs[1]
};
DocInfo *nfoptrs [2] = { &testdocset.infos[0],
&testdocset.infos[1]
};
testdocset.infos[1].content_meta = COUCH_DOC_IS_COMPRESSED; //Mark doc2 as to be snappied.
unlink(testfilepath);
Db *db;
try(couchstore_open_db(testfilepath, COUCHSTORE_OPEN_FLAG_CREATE, &db));
try(couchstore_save_documents(db, docptrs, nfoptrs, 2,
COMPRESS_DOC_BODIES));
try(couchstore_commit(db));
couchstore_close_db(db);
//Read back
try(couchstore_open_db(testfilepath, 0, &db));
try(couchstore_changes_since(db, 0, 0, docset_check, &testdocset));
assert(testdocset.counters.totaldocs == 2);
assert(testdocset.counters.deleted == 0);
couchstore_close_db(db);
cleanup:
assert(errcode == 0);
}
couchstore_error_t couchstore_save_documents(Db *db,
Doc* const docs[],
DocInfo *infos[],
unsigned numdocs,
couchstore_save_options options)
{
couchstore_error_t errcode = COUCHSTORE_SUCCESS;
unsigned ii;
sized_buf *seqklist, *idklist, *seqvlist, *idvlist;
size_t term_meta_size = 0;
const Doc *curdoc;
uint64_t seq = db->header.update_seq;
fatbuf *fb;
for (ii = 0; ii < numdocs; ii++) {
// Get additional size for terms to be inserted into indexes
// IMPORTANT: This must match the sizes of the fatbuf_get calls in add_doc_to_update_list!
term_meta_size += 6
+ 44 + infos[ii]->id.size + infos[ii]->rev_meta.size
+ 44 + 10 + infos[ii]->rev_meta.size;
}
fb = fatbuf_alloc(term_meta_size +
numdocs * (sizeof(sized_buf) * 4)); //seq/id key and value lists
if (fb == NULL) {
return COUCHSTORE_ERROR_ALLOC_FAIL;
}
seqklist = fatbuf_get(fb, numdocs * sizeof(sized_buf));
idklist = fatbuf_get(fb, numdocs * sizeof(sized_buf));
seqvlist = fatbuf_get(fb, numdocs * sizeof(sized_buf));
idvlist = fatbuf_get(fb, numdocs * sizeof(sized_buf));
for (ii = 0; ii < numdocs; ii++) {
seq++;
if (docs) {
curdoc = docs[ii];
} else {
curdoc = NULL;
}
errcode = add_doc_to_update_list(db, curdoc, infos[ii], fb,
&seqklist[ii], &idklist[ii],
&seqvlist[ii], &idvlist[ii],
seq, options);
if (errcode != COUCHSTORE_SUCCESS) {
break;
}
}
if (errcode == COUCHSTORE_SUCCESS) {
errcode = update_indexes(db, seqklist, seqvlist,
idklist, idvlist, numdocs);
}
fatbuf_free(fb);
if (errcode == COUCHSTORE_SUCCESS) {
// Fill in the assigned sequence numbers for caller's later use:
seq = db->header.update_seq;
for (ii = 0; ii < numdocs; ii++) {
infos[ii]->db_seq = ++seq;
}
db->header.update_seq = seq;
}
return errcode;
}第八个测试函数：
static void test_changes_no_dups(void)
{
fprintf(stderr, "changes no dupes... ");
fflush(stderr);
int errcode = 0;
int i;
const int numdocs = 10000;
int updatebatch = 1000;
Doc **docptrs;
DocInfo **nfoptrs;
char *docmap;
Db *db;
docset_init(numdocs);
for (i=0; i < numdocs; i++) {
char* id = malloc(100);
char* body = malloc(100);
sprintf(id, "doc%d", i);
sprintf(body, "{\"test_doc_index\":%d}", i);
setdoc(&testdocset.docs[i], &testdocset.infos[i],
id, strlen(id),
body, strlen(body),
zerometa, sizeof(zerometa));
}
docptrs = malloc(numdocs * sizeof(Doc*));
nfoptrs = malloc(numdocs * sizeof(DocInfo*));
docmap = malloc(numdocs);
for (i=0; i < numdocs; i++) {
docptrs[i] = &testdocset.docs[i];
nfoptrs[i] = &testdocset.infos[i];
}
unlink(testfilepath);
try(couchstore_open_db(testfilepath, COUCHSTORE_OPEN_FLAG_CREATE, &db));
// only save half the docs at first.
try(couchstore_save_documents(db, docptrs, nfoptrs, numdocs/2, 0));
try(couchstore_commit(db));
couchstore_close_db(db);
for (i=0; i < numdocs/2; i++) {
// increment the rev for already added docs
nfoptrs[i]->rev_seq++;
}
srand(10); // make deterministic
// now shuffle so some bulk updates contain previous docs and new docs
shuffle(docptrs, nfoptrs, numdocs);
try(couchstore_open_db(testfilepath, 0, &db));
for (i=0; i < numdocs; i += updatebatch) {
// now do bulk updates and check the changes for dups
try(couchstore_save_documents(db, docptrs + i, nfoptrs + i, updatebatch, 0));
try(couchstore_commit(db));
memset(docmap, 0, numdocs);
try(couchstore_changes_since(db, 0, 0, docmap_check, docmap));
}
DbInfo info;
assert(couchstore_db_info(db, &info) == COUCHSTORE_SUCCESS);
assert(info.last_sequence == (uint64_t)(numdocs + numdocs/2));
assert(info.doc_count == (uint64_t)numdocs);
assert(info.deleted_count == 0);
couchstore_close_db(db);
cleanup:
for (i=0; i < numdocs; i++) {
free(docptrs[i]->id.buf);
free(docptrs[i]->data.buf);
}
free(docptrs);
free(nfoptrs);
free(docmap);
assert(errcode == 0);
}
这个函数我看着有点晕，怎么看都是检验seq值是否可以在文档更新后保证递增，也就是保证seq的正确性，别的作用没看出来。
第九个卖萌函数：
static void mb5086(void)
{
Db *db;
Doc d;
DocInfo i;
couchstore_error_t err;
fprintf(stderr, "regression mb-5086.... ");
fflush(stderr);
setdoc(&d, &i, "hi", 2, "foo", 3, NULL, 0);
err = couchstore_open_db("mb5085.couch", COUCHSTORE_OPEN_FLAG_CREATE, &db);
assert(err == COUCHSTORE_SUCCESS);
assert(couchstore_save_document(db, &d, &i, 0) == COUCHSTORE_SUCCESS);
assert(couchstore_commit(db) == COUCHSTORE_SUCCESS);
assert(couchstore_close_db(db) == COUCHSTORE_SUCCESS);
assert(remove("mb5085.couch") == 0);
}第十个检测函数：
static void test_huge_revseq(void)
{
Db *db;
Doc d;
DocInfo i;
DocInfo *i2;
couchstore_error_t err;
fprintf(stderr, "huge rev_seq.... ");
fflush(stderr);
setdoc(&d, &i, "hi", 2, "foo", 3, NULL, 0);
i.rev_seq = 5294967296;
err = couchstore_open_db("bigrevseq.couch", COUCHSTORE_OPEN_FLAG_CREATE, &db);
assert(err == COUCHSTORE_SUCCESS);
assert(couchstore_save_document(db, &d, &i, 0) == COUCHSTORE_SUCCESS);
assert(couchstore_commit(db) == COUCHSTORE_SUCCESS);
assert(couchstore_docinfo_by_id(db, "hi", 2, &i2) == COUCHSTORE_SUCCESS);
assert(i2->rev_seq == 5294967296);
couchstore_free_docinfo(i2);
assert(couchstore_close_db(db) == COUCHSTORE_SUCCESS);
assert(remove("bigrevseq.couch") == 0);
}也就是检测一下seq值在数值巨大的情况下是否正常，不过seq是uint64_t类型的，所以实际上感觉对于seq本身这数字不算大，大概主要是测试在处理过程中不会在中间过程出问题吧，等以后具体看底层实现就知道是不是这回事了。
第十一个检测函数：
void TestCollateJSON(void)
{
fprintf(stderr, "JSON collation: ");
TestCollateConvertEscape();
TestCollateScalars();
TestCollateASCII();
TestCollateRaw();
TestCollateArrays();
TestCollateNestedArrays();
TestCollateUnicodeStrings();
fprintf(stderr, "OK\n");
}
第十二个检测函数：
void TestCouchIndexer(void) {
fprintf(stderr, "Indexer: ");
srandom(42); // to get a consistent sequence of random numbers
GenerateKVFile(KVPATH, 1000);
srandom(42); // to get a consistent sequence of random numbers
GenerateBackIndexKVFile(KVBACKPATH, 1000);
IndexKVFile(KVPATH, KVBACKPATH, INDEXPATH, COUCHSTORE_REDUCE_STATS);
ReadIndexFile(INDEXPATH);
unlink(KVPATH);
unlink(INDEXPATH);
fprintf(stderr, "OK\n");
}
扫一扫
3276
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
