1. report流程简介
report行为是storage向tracker注册storage，上报storage信息的行为。还需注意的是：它根据返回结果触发同步sync。
其storage端的流程大概如下：
- 在main[storage/fdfs_storaged.c]函数中，初始化：tracker_report_init[storage/tracker_client_thread.c]。其功能为：清空g_storage_servers，g_sorted_storages，和初始化report线程锁。【见2.1】
- 在main[storage/fdfs_storaged.c]函数中，触发创建report线程tracker_report_thread_start[storage/tracker_client_thread.c]。
- 在tracker_report_thread_start函数中，根据配置文件的tracker配置，为每一个tracker创建一个report线程tracker_report_thread_entrance [storage/fdfs_storaged.c]。【见2.2】
- 在tracker_report_thread_entrance函数中，先调用tracker_report_join向tracker注册storage，再周期性上报storage status，磁盘信息等。【见2.3】
- 在上报信息的函数中，得到tracker的返回结果。根据返回结果触发同步sync线程。
2. storage report相关源代码分析
2.1 tracker_report_init函数
report初始化函数。其功能比较简单，清空g_storage_servers，g_sorted_storages，和初始化report线程锁。
storage/tracker_client_thread.c
int tracker_report_init()
{
int result;
// 清空g_storage_servers，保存storage信息
memset(g_storage_servers, 0, sizeof(g_storage_servers));
// 清空g_sorted_storages，排序
memset(g_sorted_storages, 0, sizeof(g_sorted_storages));
// 初始化report线程锁
if ((result=init_pthread_lock(&reporter_thread_lock)) != 0)
{
return result;
}
return 0;
}
2.2 tracker_report_thread_start函数
创建report线程函数，根据配置，为每一个tracker都创建一个report线程。
storage/tracker_client_thread.c
int tracker_report_thread_start()
{
ConnectionInfo *pTrackerServer;
ConnectionInfo *pServerEnd;
pthread_attr_t pattr;
pthread_t tid;
int result;
// 线程属性
if ((result=init_pthread_attr(&pattr, g_thread_stack_size)) != 0)
{
return result;
}
// 线程pthread_t
report_tids = (pthread_t *)malloc(sizeof(pthread_t) * \
g_tracker_group.server_count);
if (report_tids == NULL)
{
logError("file: "__FILE__", line: %d, " \
"malloc %d bytes fail, " \
"errno: %d, error info: %s", \
__LINE__, (int)sizeof(pthread_t) * \
g_tracker_group.server_count, \
errno, STRERROR(errno));
return errno != 0 ? errno : ENOMEM;
}
memset(report_tids, 0, sizeof(pthread_t)*g_tracker_group.server_count);
// tracker server状态，由向tracker注册时返回。
src_storage_status = (int *)malloc(sizeof(int) * \
g_tracker_group.server_count);
if (src_storage_status == NULL)
{
logError("file: "__FILE__", line: %d, " \
"malloc %d bytes fail, " \
"errno: %d, error info: %s", __LINE__, \
(int)sizeof(int) * g_tracker_group.server_count, \
errno, STRERROR(errno));
return errno != 0 ? errno : ENOMEM;
}
memset(src_storage_status,-1,sizeof(int)*g_tracker_group.server_count);
// report的状态，由向tracker注册时返回。
my_report_status = (signed char *)malloc(sizeof(signed char) * \
g_tracker_group.server_count);
if (my_report_status == NULL)
{
logError("file: "__FILE__", line: %d, " \
"malloc %d bytes fail, " \
"errno: %d, error info: %s", __LINE__, \
(int)sizeof(signed char) * g_tracker_group.server_count, \
errno, STRERROR(errno));
return errno != 0 ? errno : ENOMEM;
}
memset(my_report_status, -1, sizeof(char)*g_tracker_group.server_count);
// 创建report线程，并记录线程总数
g_tracker_reporter_count = 0;
pServerEnd = g_tracker_group.servers + g_tracker_group.server_count;
for (pTrackerServer=g_tracker_group.servers; pTrackerServer<pServerEnd; \
pTrackerServer++)
{
// 创建线程，线程函数：tracker_report_thread_entrance[见2.3]
if((result=pthread_create(&tid, &pattr, \
tracker_report_thread_entrance, pTrackerServer)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"create thread failed, errno: %d, " \
"error info: %s.", \
__LINE__, result, STRERROR(result));
return result;
}
// 更新全局变量
if ((result=pthread_mutex_lock(&reporter_thread_lock)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"call pthread_mutex_lock fail, " \
"errno: %d, error info: %s", \
__LINE__, result, STRERROR(result));
}
report_tids[g_tracker_reporter_count] = tid;
g_tracker_reporter_count++;
if ((result=pthread_mutex_unlock(&reporter_thread_lock)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"call pthread_mutex_unlock fail, " \
"errno: %d, error info: %s", \
__LINE__, result, STRERROR(result));
}
}
pthread_attr_destroy(&pattr);
return 0;
}
2.3 tracker_report_thread_entrance函数
线程执行函数。先向tracker注册storage，再周期性上报storage相关信息。
storage/tracker_client_thread.c
static void *tracker_report_thread_entrance(void *arg)
{
ConnectionInfo *pTrackerServer;
char my_server_id[FDFS_STORAGE_ID_MAX_SIZE];
char tracker_client_ip[IP_ADDRESS_SIZE];
char szFailPrompt[36];
bool sync_old_done;
int stat_chg_sync_count;
int sync_time_chg_count;
time_t current_time;
time_t last_df_report_time;
time_t last_sync_report_time;
time_t last_beat_time;
int last_trunk_file_id;
int result;
int previousCode;
int nContinuousFail;
int tracker_index;
int64_t last_trunk_total_free_space;
bool bServerPortChanged;
// storage 端口号是否有变化
bServerPortChanged = (g_last_server_port != 0) && \
(g_server_port != g_last_server_port);
// 1.获取tracker连接信息
pTrackerServer = (ConnectionInfo *)arg;
pTrackerServer->sock = -1;
// tracker位置标志
tracker_index = pTrackerServer - g_tracker_group.servers;
logDebug("file: "__FILE__", line: %d, " \
"report thread to tracker server %s:%d started", \
__LINE__, pTrackerServer->ip_addr, pTrackerServer->port);
// 同步完成标志
sync_old_done = g_sync_old_done;
// 等待所有线程都创建、启动
while (g_continue_flag && \
g_tracker_reporter_count < g_tracker_group.server_count)
{
sleep(1); //waiting for all thread started
}
result = 0;
previousCode = 0;
nContinuousFail = 0;
while (g_continue_flag)
{
// 2.连接tracker
if (pTrackerServer->sock >= 0)
{
close(pTrackerServer->sock);
}
pTrackerServer->sock = socket(AF_INET, SOCK_STREAM, 0);
if(pTrackerServer->sock < 0)
{
logCrit("file: "__FILE__", line: %d, " \
"socket create failed, errno: %d, " \
"error info: %s. program exit!", \
__LINE__, errno, STRERROR(errno));
g_continue_flag = false;
break;
}
if (g_client_bind_addr && *g_bind_addr != '\0')
{
socketBind(pTrackerServer->sock, g_bind_addr, 0);
}
tcpsetserveropt(pTrackerServer->sock, g_fdfs_network_timeout);
if (tcpsetnonblockopt(pTrackerServer->sock) != 0)
{
nContinuousFail++;
sleep(g_heart_beat_interval);
continue;
}
if ((result=connectserverbyip_nb(pTrackerServer->sock, \
pTrackerServer->ip_addr, \
pTrackerServer->port, g_fdfs_connect_timeout)) != 0)
{
if (previousCode != result)
{
logError("file: "__FILE__", line: %d, " \
"connect to tracker server %s:%d fail" \
", errno: %d, error info: %s", \
__LINE__, pTrackerServer->ip_addr, \
pTrackerServer->port, \
result, STRERROR(result));
previousCode = result;
}
nContinuousFail++;
if (g_continue_flag)
{
sleep(g_heart_beat_interval);
continue;
}
else
{
break;
}
}
// 2. 获取tracker的网卡ip地址tracker_client_ip
getSockIpaddr(pTrackerServer->sock, \
tracker_client_ip, IP_ADDRESS_SIZE);
if (nContinuousFail == 0)
{
*szFailPrompt = '\0';
}
else
{
sprintf(szFailPrompt, ", continuous fail count: %d", \
nContinuousFail);
}
logInfo("file: "__FILE__", line: %d, " \
"successfully connect to tracker server %s:%d%s, " \
"as a tracker client, my ip is %s", \
__LINE__, pTrackerServer->ip_addr, \
pTrackerServer->port, szFailPrompt, tracker_client_ip);
previousCode = 0;
nContinuousFail = 0;
// 判断tracker_client_ip是否正确
if (*g_tracker_client_ip == '\0')
{
strcpy(g_tracker_client_ip, tracker_client_ip);
}
else if (strcmp(tracker_client_ip, g_tracker_client_ip) != 0)
{
logError("file: "__FILE__", line: %d, " \
"as a client of tracker server %s:%d, " \
"my ip: %s != client ip: %s of other " \
"tracker client", __LINE__, \
pTrackerServer->ip_addr, pTrackerServer->port, \
tracker_client_ip, g_tracker_client_ip);
close(pTrackerServer->sock);
pTrackerServer->sock = -1;
break;
}
// 记录storage本地ip
insert_into_local_host_ip(tracker_client_ip);
/*
//printf("file: "__FILE__", line: %d, " \
"tracker_client_ip: %s, g_my_server_id_str: %s\n", \
__LINE__, tracker_client_ip, g_my_server_id_str);
//print_local_host_ip_addrs();
*/
// 3.向tracker注册storage，包括storage配置和storage运行状态。[见2.4]
if (tracker_report_join(pTrackerServer, tracker_index, \
sync_old_done) != 0)
{
sleep(g_heart_beat_interval);
continue;
}
// 如果端口号改变，更新文件.data_init_flag
if (g_http_port != g_last_http_port)
{
g_last_http_port = g_http_port;
if ((result=storage_write_to_sync_ini_file()) != 0)
{
}
}
// 如果需要同步
if (!sync_old_done)
{
// 加report线程锁，为了线程同步
if ((result=pthread_mutex_lock(&reporter_thread_lock)) \
!= 0)
{
logError("file: "__FILE__", line: %d, " \
"call pthread_mutex_lock fail, " \
"errno: %d, error info: %s", \
__LINE__, result, STRERROR(result));
fdfs_quit(pTrackerServer);
sleep(g_heart_beat_interval);
continue;
}
if (!g_sync_old_done)
{
// 询问tracker，获取 src storage id and time stamp【见2.6】
if (tracker_sync_dest_req(pTrackerServer) == 0)
{
g_sync_old_done = true;
// 更新sync信息到.data_init_flag
if (storage_write_to_sync_ini_file() \
!= 0)
{
logCrit("file: "__FILE__", line: %d, " \
"storage_write_to_sync_ini_file"\
" fail, program exit!", \
__LINE__);
g_continue_flag = false;
pthread_mutex_unlock( \
&reporter_thread_lock);
break;
}
}
else //request failed or need to try again
{
pthread_mutex_unlock( \
&reporter_thread_lock);
fdfs_quit(pTrackerServer);
sleep(g_heart_beat_interval);
continue;
}
}
else
{
// 向tracker通知sync's src storage id and time stamp【见2.5】
if (tracker_sync_notify(pTrackerServer, tracker_index) != 0)
{
pthread_mutex_unlock( \
&reporter_thread_lock);
fdfs_quit(pTrackerServer);
sleep(g_heart_beat_interval);
continue;
}
}
if ((result=pthread_mutex_unlock(&reporter_thread_lock))
!= 0)
{
logError("file: "__FILE__", line: %d, " \
"call pthread_mutex_unlock fail, " \
"errno: %d, error info: %s", \
__LINE__, result, STRERROR(result));
}
sync_old_done = true;
}
// 4.向tracker通知sync's src storage id and time stamp，并记录是否成功【见2.5】
src_storage_status[tracker_index] = \
tracker_sync_notify(pTrackerServer, tracker_index);
if (src_storage_status[tracker_index] != 0)
{// 如果不成功，检测是否有成功的tracker。
int k;
for (k=0; k<g_tracker_group.server_count; k++)
{
if (src_storage_status[k] != ENOENT)
{
break;
}
}
// 如果没有，
if (k == g_tracker_group.server_count)
{ //src storage server already be deleted
int my_status;
if (tracker_get_storage_max_status( \
&g_tracker_group, g_group_name, \
g_tracker_client_ip, my_server_id, \
&my_status) == 0)
{
tracker_sync_dest_query(pTrackerServer); // [见2.6]
if(my_status<FDFS_STORAGE_STATUS_OFFLINE
&& g_sync_old_done)
{ //need re-sync old files
pthread_mutex_lock( \
&reporter_thread_lock);
g_sync_old_done = false;
sync_old_done = g_sync_old_done;
storage_write_to_sync_ini_file();
pthread_mutex_unlock( \
&reporter_thread_lock);
}
}
}
fdfs_quit(pTrackerServer);
sleep(g_heart_beat_interval);
continue;
}
sync_time_chg_count = 0;
last_df_report_time = 0;
last_beat_time = 0;
last_sync_report_time = 0;
stat_chg_sync_count = 0;
last_trunk_file_id = 0;
last_trunk_total_free_space = -1;
// 5. 周期上报信息
while (g_continue_flag)
{
current_time = g_current_time;
if (current_time - last_beat_time >= \
g_heart_beat_interval)
{
// 5.1 心跳包，上报storage状态 [见2.7]
if (tracker_heart_beat(pTrackerServer, \
&stat_chg_sync_count, \
&bServerPortChanged) != 0)
{
break;
}
if (g_storage_ip_changed_auto_adjust && \
tracker_storage_changelog_req( \
pTrackerServer) != 0)
{
break;
}
last_beat_time = current_time;
}
if (sync_time_chg_count != g_sync_change_count && \
current_time - last_sync_report_time >= \
g_heart_beat_interval)
{
// 5.2 上报最新的同步时间戳 [见2.8]
if (tracker_report_sync_timestamp( \
pTrackerServer, &bServerPortChanged)!=0)
{
break;
}
sync_time_chg_count = g_sync_change_count;
last_sync_report_time = current_time;
}
if (current_time - last_df_report_time >= \
g_stat_report_interval)
{
// 5.3 上报storage挂载点磁盘状态 [见2.9]
if (tracker_report_df_stat(pTrackerServer, \
&bServerPortChanged) != 0)
{
break;
}
last_df_report_time = current_time;
}
// 如果使用“合并存储”
if (g_if_trunker_self)
{
if (last_trunk_file_id < g_current_trunk_file_id)
{
if (tracker_report_trunk_fid(pTrackerServer)!=0)
{
break;
}
last_trunk_file_id = g_current_trunk_file_id;
}
if (last_trunk_total_free_space != g_trunk_total_free_space)
{
if (tracker_report_trunk_free_space(pTrackerServer)!=0)
{
break;
}
last_trunk_total_free_space = g_trunk_total_free_space;
}
}
// 判读是否需要重新注册
if (need_rejoin_tracker)
{
need_rejoin_tracker = false;
break;
}
sleep(1);
} // 周期性上报信息，结束
if ((!g_continue_flag) && fdfs_quit(pTrackerServer) != 0)
{
}
close(pTrackerServer->sock);
pTrackerServer->sock = -1;
if (g_continue_flag)
{
sleep(1);
}
} // 线程 while 结束
if (nContinuousFail > 0)
{
logError("file: "__FILE__", line: %d, " \
"connect to tracker server %s:%d fail, try count: %d" \
", errno: %d, error info: %s", \
__LINE__, pTrackerServer->ip_addr, \
pTrackerServer->port, nContinuousFail, \
result, STRERROR(result));
}
thracker_report_thread_exit(pTrackerServer);
return NULL;
}
2.4 tracker_report_join函数
向tracker注册storage的函数。
storage/tracker_client_thread.c
int tracker_report_join(ConnectionInfo *pTrackerServer, \
const int tracker_index, const bool sync_old_done)
{
char out_buff[sizeof(TrackerHeader) + sizeof(TrackerStorageJoinBody) + \
FDFS_MAX_TRACKERS * FDFS_PROTO_IP_PORT_SIZE];
TrackerHeader *pHeader;
TrackerStorageJoinBody *pReqBody;
TrackerStorageJoinBodyResp respBody;
char *pInBuff;
char *p;
ConnectionInfo *pServer;
ConnectionInfo *pServerEnd;
FDFSStorageServer *pTargetServer;
FDFSStorageServer **ppFound;
FDFSStorageServer targetServer;
int out_len;
//int tracker_count;
int result;
int i;
int64_t in_bytes;
// 1.封装注册上报信息
pHeader = (TrackerHeader *)out_buff;
pReqBody = (TrackerStorageJoinBody *)(out_buff+sizeof(TrackerHeader));
memset(out_buff, 0, sizeof(out_buff));
pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_JOIN; //【见3.1】
strcpy(pReqBody->group_name, g_group_name);
strcpy(pReqBody->domain_name, g_http_domain);
snprintf(pReqBody->version, sizeof(pReqBody->version), "%d.%02d", \
g_fdfs_version.major, g_fdfs_version.minor);
long2buff(g_server_port, pReqBody->storage_port);
long2buff(g_http_port, pReqBody->storage_http_port);
long2buff(g_fdfs_store_paths.count, pReqBody->store_path_count);
long2buff(g_subdir_count_per_path, pReqBody->subdir_count_per_path);
long2buff(g_upload_priority, pReqBody->upload_priority);
long2buff(g_storage_join_time, pReqBody->join_time);
long2buff(g_up_time, pReqBody->up_time);
pReqBody->init_flag = sync_old_done ? 0 : 1;
// 封装stoage当前运行状态
memset(&targetServer, 0, sizeof(targetServer));
pTargetServer = &targetServer;
strcpy(targetServer.server.id, g_my_server_id_str);
ppFound = (FDFSStorageServer **)bsearch(&pTargetServer, \
g_sorted_storages, g_storage_count, \
sizeof(FDFSStorageServer *), storage_cmp_by_server_id);
if (ppFound != NULL)
{
pReqBody->status = (*ppFound)->server.status;
}
else
{
if (g_tracker_group.server_count > 1)
{
for (i=0; i<g_tracker_group.server_count; i++)
{
if (my_report_status[i] == -1)
{
logInfo("file: "__FILE__", line: %d, "
"tracker server: #%d. %s:%d, my_report_status: %d",
__LINE__, i, g_tracker_group.servers[i].ip_addr,
g_tracker_group.servers[i].port, my_report_status[i]);
break;
}
}
if (i == g_tracker_group.server_count)
{
pReqBody->status = FDFS_STORAGE_STATUS_INIT;
}
else
{
pReqBody->status = -1;
}
}
else
{
pReqBody->status = FDFS_STORAGE_STATUS_INIT;
}
}
// 封装storage配置的trackers
//tracker_count = 0;
p = out_buff + sizeof(TrackerHeader) + sizeof(TrackerStorageJoinBody);
pServerEnd = g_tracker_group.servers + g_tracker_group.server_count;
for (pServer=g_tracker_group.servers; pServer<pServerEnd; pServer++)
{
/*
if (strcmp(pServer->ip_addr, pTrackerServer->ip_addr) == 0 && \
pServer->port == pTrackerServer->port)
{
continue;
}
tracker_count++;
*/
sprintf(p, "%s:%d", pServer->ip_addr, pServer->port);
p += FDFS_PROTO_IP_PORT_SIZE;
}
out_len = p - out_buff;
long2buff(g_tracker_group.server_count, pReqBody->tracker_count);
long2buff(out_len - (int)sizeof(TrackerHeader), pHeader->pkg_len);
// 2.发送封装好信息
if ((result=tcpsenddata_nb(pTrackerServer->sock, out_buff, \
out_len, g_fdfs_network_timeout)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"tracker server %s:%d, send data fail, " \
"errno: %d, error info: %s.", \
__LINE__, pTrackerServer->ip_addr, \
pTrackerServer->port, \
result, STRERROR(result));
return result;
}
// 3.接收返回信息
pInBuff = (char *)&respBody;
result = fdfs_recv_response(pTrackerServer, \
&pInBuff, sizeof(respBody), &in_bytes);
my_report_status[tracker_index] = result;
if (result != 0)
{
logError("file: "__FILE__", line: %d, "
"fdfs_recv_response fail, result: %d",
__LINE__, result);
return result;
}
if (in_bytes != sizeof(respBody))
{
logError("file: "__FILE__", line: %d, " \
"tracker server %s:%d, recv data fail, " \
"expect %d bytes, but recv " \
"%"PRId64" bytes", \
__LINE__, pTrackerServer->ip_addr, \
pTrackerServer->port, \
(int)sizeof(respBody), in_bytes);
my_report_status[tracker_index] = EINVAL;
return EINVAL;
}
if (*(respBody.src_id) == '\0' && *g_sync_src_id != '\0')
{// 如果返回信息没有src_id信息，而本地存在，则上报
// pxxian: at this time, notify src storage id.
return tracker_sync_notify(pTrackerServer, tracker_index);
}
else
{
return 0;
}
}
2.5 tracker_sync_notify函数
上报src_id和同步时间戳的函数。
storage/tracker_client_thread.c
static int tracker_sync_notify(ConnectionInfo *pTrackerServer, const int tracker_index)
{
char out_buff[sizeof(TrackerHeader)+sizeof(TrackerStorageSyncReqBody)];
TrackerHeader *pHeader;
TrackerStorageSyncReqBody *pReqBody;
int64_t in_bytes;
int result;
// 1.封装上报信息：g_sync_src_id，g_sync_until_timestamp
pHeader = (TrackerHeader *)out_buff;
pReqBody = (TrackerStorageSyncReqBody*)(out_buff+sizeof(TrackerHeader));
memset(out_buff, 0, sizeof(out_buff));
long2buff((int)sizeof(TrackerStorageSyncReqBody), pHeader->pkg_len);
pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_SYNC_NOTIFY;//【见3.2】
strcpy(pReqBody->src_id, g_sync_src_id);
long2buff(g_sync_until_timestamp, pReqBody->until_timestamp);
// 2.发送上报信息
if ((result=tcpsenddata_nb(pTrackerServer->sock, out_buff, \
sizeof(out_buff), g_fdfs_network_timeout)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"tracker server %s:%d, send data fail, " \
"errno: %d, error info: %s.", \
__LINE__, pTrackerServer->ip_addr, \
pTrackerServer->port, \
result, STRERROR(result));
return result;
}
// 3. 接收反馈信息
if ((result=fdfs_recv_header(pTrackerServer, &in_bytes)) != 0)
{
// no such file or directory
if (result == ENOENT)
{
// leader server index
if (g_tracker_group.leader_index == -1)
{
get_tracker_leader();
}
if (tracker_index == g_tracker_group.leader_index)
{
logWarning("file: "__FILE__", line: %d, "
"clear sync src id: %s because "
"tracker leader response ENOENT",
__LINE__, g_sync_src_id);
*g_sync_src_id = '\0';
storage_write_to_sync_ini_file();
}
}
if (result != 0 && result != ENOENT)
{
logError("file: "__FILE__", line: %d, "
"fdfs_recv_header fail, result: %d",
__LINE__, result);
return result;
}
}
if (in_bytes != 0)
{
logError("file: "__FILE__", line: %d, " \
"tracker server %s:%d, recv body length: " \
"%"PRId64" != 0", \
__LINE__, pTrackerServer->ip_addr, \
pTrackerServer->port, in_bytes);
return EINVAL;
}
return result;
}
2.6 tracker_sync_dest_req函数
请求同步信息函数。获取g_sync_src_id，g_sync_until_timestamp
storage/tracker_client_thread.c
static int tracker_sync_dest_req(ConnectionInfo *pTrackerServer)
{
TrackerHeader header;
TrackerStorageSyncReqBody syncReqbody;
char *pBuff;
int64_t in_bytes;
int result;
// 1.发送请求
memset(&header, 0, sizeof(header));
header.cmd = TRACKER_PROTO_CMD_STORAGE_SYNC_DEST_REQ;//【见3.3】
if ((result=tcpsenddata_nb(pTrackerServer->sock, &header, \
sizeof(header), g_fdfs_network_timeout)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"tracker server %s:%d, send data fail, " \
"errno: %d, error info: %s.", \
__LINE__, pTrackerServer->ip_addr, \
pTrackerServer->port, \
result, STRERROR(result));
return result;
}
// 2.接收反馈
pBuff = (char *)&syncReqbody;
if ((result=fdfs_recv_response(pTrackerServer, \
&pBuff, sizeof(syncReqbody), &in_bytes)) != 0)
{
logError("file: "__FILE__", line: %d, "
"fdfs_recv_response fail, result: %d",
__LINE__, result);
return result;
}
if (in_bytes == 0)
{
return result;
}
if (in_bytes != sizeof(syncReqbody))
{
logError("file: "__FILE__", line: %d, " \
"tracker server %s:%d, " \
"recv body length: %"PRId64" is invalid, " \
"expect body length: %d", \
__LINE__, pTrackerServer->ip_addr, \
pTrackerServer->port, in_bytes, \
(int)sizeof(syncReqbody));
return EINVAL;
}
// 3.解析反馈信息，获取g_sync_src_id，g_sync_until_timestamp
memcpy(g_sync_src_id, syncReqbody.src_id, FDFS_STORAGE_ID_MAX_SIZE);
g_sync_src_id[FDFS_STORAGE_ID_MAX_SIZE - 1] = '\0';
g_sync_until_timestamp = (time_t)buff2long(syncReqbody.until_timestamp);
return 0;
}
2.7 tracker_heart_beat函数
周期性上报信息的心跳函数。主要上报storage status。
storage/tracker_client_thread.c
static int tracker_heart_beat(ConnectionInfo *pTrackerServer, \
int *pstat_chg_sync_count, bool *bServerPortChanged)
{
char out_buff[sizeof(TrackerHeader) + sizeof(FDFSStorageStatBuff)];
TrackerHeader *pHeader;
FDFSStorageStatBuff *pStatBuff;
int body_len;
int result;
// 1.封装发送信息FDFSStorageStatBuff
memset(out_buff, 0, sizeof(out_buff));
pHeader = (TrackerHeader *)out_buff;
// 判断是否有变化，无变化不发送
if (*pstat_chg_sync_count != g_stat_change_count)
{
pStatBuff = (FDFSStorageStatBuff *)( \
out_buff + sizeof(TrackerHeader));
int2buff(free_queue_alloc_connections(), \
pStatBuff->connection.sz_alloc_count);
int2buff(g_storage_stat.connection.current_count, \
pStatBuff->connection.sz_current_count);
int2buff(g_storage_stat.connection.max_count, \
pStatBuff->connection.sz_max_count);
long2buff(g_storage_stat.total_upload_count, \
pStatBuff->sz_total_upload_count);
long2buff(g_storage_stat.success_upload_count, \
pStatBuff->sz_success_upload_count);
long2buff(g_storage_stat.total_append_count, \
pStatBuff->sz_total_append_count);
long2buff(g_storage_stat.success_append_count, \
pStatBuff->sz_success_append_count);
long2buff(g_storage_stat.total_modify_count, \
pStatBuff->sz_total_modify_count);
long2buff(g_storage_stat.success_modify_count, \
pStatBuff->sz_success_modify_count);
long2buff(g_storage_stat.total_truncate_count, \
pStatBuff->sz_total_truncate_count);
long2buff(g_storage_stat.success_truncate_count, \
pStatBuff->sz_success_truncate_count);
long2buff(g_storage_stat.total_download_count, \
pStatBuff->sz_total_download_count);
long2buff(g_storage_stat.success_download_count, \
pStatBuff->sz_success_download_count);
long2buff(g_storage_stat.total_set_meta_count, \
pStatBuff->sz_total_set_meta_count);
long2buff(g_storage_stat.success_set_meta_count, \
pStatBuff->sz_success_set_meta_count);
long2buff(g_storage_stat.total_delete_count, \
pStatBuff->sz_total_delete_count);
long2buff(g_storage_stat.success_delete_count, \
pStatBuff->sz_success_delete_count);
long2buff(g_storage_stat.total_get_meta_count, \
pStatBuff->sz_total_get_meta_count);
long2buff(g_storage_stat.success_get_meta_count, \
pStatBuff->sz_success_get_meta_count);
long2buff(g_storage_stat.total_create_link_count, \
pStatBuff->sz_total_create_link_count);
long2buff(g_storage_stat.success_create_link_count, \
pStatBuff->sz_success_create_link_count);
long2buff(g_storage_stat.total_delete_link_count, \
pStatBuff->sz_total_delete_link_count);
long2buff(g_storage_stat.success_delete_link_count, \
pStatBuff->sz_success_delete_link_count);
long2buff(g_storage_stat.total_upload_bytes, \
pStatBuff->sz_total_upload_bytes);
long2buff(g_storage_stat.success_upload_bytes, \
pStatBuff->sz_success_upload_bytes);
long2buff(g_storage_stat.total_append_bytes, \
pStatBuff->sz_total_append_bytes);
long2buff(g_storage_stat.success_append_bytes, \
pStatBuff->sz_success_append_bytes);
long2buff(g_storage_stat.total_modify_bytes, \
pStatBuff->sz_total_modify_bytes);
long2buff(g_storage_stat.success_modify_bytes, \
pStatBuff->sz_success_modify_bytes);
long2buff(g_storage_stat.total_download_bytes, \
pStatBuff->sz_total_download_bytes);
long2buff(g_storage_stat.success_download_bytes, \
pStatBuff->sz_success_download_bytes);
long2buff(g_storage_stat.total_sync_in_bytes, \
pStatBuff->sz_total_sync_in_bytes);
long2buff(g_storage_stat.success_sync_in_bytes, \
pStatBuff->sz_success_sync_in_bytes);
long2buff(g_storage_stat.total_sync_out_bytes, \
pStatBuff->sz_total_sync_out_bytes);
long2buff(g_storage_stat.success_sync_out_bytes, \
pStatBuff->sz_success_sync_out_bytes);
long2buff(g_storage_stat.total_file_open_count, \
pStatBuff->sz_total_file_open_count);
long2buff(g_storage_stat.success_file_open_count, \
pStatBuff->sz_success_file_open_count);
long2buff(g_storage_stat.total_file_read_count, \
pStatBuff->sz_total_file_read_count);
long2buff(g_storage_stat.success_file_read_count, \
pStatBuff->sz_success_file_read_count);
long2buff(g_storage_stat.total_file_write_count, \
pStatBuff->sz_total_file_write_count);
long2buff(g_storage_stat.success_file_write_count, \
pStatBuff->sz_success_file_write_count);
long2buff(g_storage_stat.last_source_update, \
pStatBuff->sz_last_source_update);
long2buff(g_storage_stat.last_sync_update, \
pStatBuff->sz_last_sync_update);
*pstat_chg_sync_count = g_stat_change_count;
body_len = sizeof(FDFSStorageStatBuff);
}
else
{
body_len = 0;
}
// 封装信息头
long2buff(body_len, pHeader->pkg_len);
pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_BEAT;//【见3.4】
// 2.发送
if((result=tcpsenddata_nb(pTrackerServer->sock, out_buff, \
sizeof(TrackerHeader) + body_len, g_fdfs_network_timeout)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"tracker server %s:%d, send data fail, " \
"errno: %d, error info: %s.", \
__LINE__, pTrackerServer->ip_addr, \
pTrackerServer->port, \
result, STRERROR(result));
return result;
}
// 3.接收，根据变化，触发文件同步
return tracker_check_response(pTrackerServer, bServerPortChanged);
}
2.8 tracker_report_sync_timestamp函数
上报与同组storage同步的时间戳。
storage/tracker_client_thread.c
static int tracker_report_sync_timestamp(ConnectionInfo *pTrackerServer, \
bool *bServerPortChanged)
{
char out_buff[sizeof(TrackerHeader) + (FDFS_STORAGE_ID_MAX_SIZE + 4) * \
FDFS_MAX_SERVERS_EACH_GROUP];
char *p;
TrackerHeader *pHeader;
FDFSStorageServer *pServer;
FDFSStorageServer *pEnd;
int result;
int body_len;
if (g_storage_count == 0)
{
return 0;
}
// 1.封装上报storage记录的同步的storages ip_addr和最后同步时间戳
memset(out_buff, 0, sizeof(out_buff));
pHeader = (TrackerHeader *)out_buff;
p = out_buff + sizeof(TrackerHeader);
body_len = (FDFS_STORAGE_ID_MAX_SIZE + 4) * g_storage_count;
pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_SYNC_REPORT;//【见3.5】
long2buff(body_len, pHeader->pkg_len);
pEnd = g_storage_servers + g_storage_count;
for (pServer=g_storage_servers; pServer<pEnd; pServer++)
{
memcpy(p, pServer->server.id, FDFS_STORAGE_ID_MAX_SIZE);
p += FDFS_STORAGE_ID_MAX_SIZE;
int2buff(pServer->last_sync_src_timestamp, p);
p += 4;
}
// 2.发送
if((result=tcpsenddata_nb(pTrackerServer->sock, out_buff, \
sizeof(TrackerHeader) + body_len, g_fdfs_network_timeout)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"tracker server %s:%d, send data fail, " \
"errno: %d, error info: %s.", \
__LINE__, pTrackerServer->ip_addr, \
pTrackerServer->port, \
result, STRERROR(result));
return result;
}
// 3.接收反馈。根据变化，触发文件同步sync
return tracker_check_response(pTrackerServer, bServerPortChanged);
}
2.9 tracker_report_df_stat函数
上报挂载点磁盘信息。如果有一个挂载点无法访问，即不上报。
storage/tracker_client_thread.c
static int tracker_report_df_stat(ConnectionInfo *pTrackerServer, \
bool *bServerPortChanged)
{
char out_buff[sizeof(TrackerHeader) + \
sizeof(TrackerStatReportReqBody) * 16];
char *pBuff;
TrackerHeader *pHeader;
TrackerStatReportReqBody *pStatBuff;
struct statvfs sbuf;
int body_len;
int total_len;
int store_path_index;
int i;
int result;
// 准备上报信息buff
body_len = (int)sizeof(TrackerStatReportReqBody) * g_fdfs_store_paths.count;
total_len = (int)sizeof(TrackerHeader) + body_len;
if (total_len <= sizeof(out_buff))
{
pBuff = out_buff;
}
else
{
pBuff = (char *)malloc(total_len);
if (pBuff == NULL)
{
logError("file: "__FILE__", line: %d, " \
"malloc %d bytes fail, " \
"errno: %d, error info: %s", \
__LINE__, total_len, \
errno, STRERROR(errno));
return errno != 0 ? errno : ENOMEM;
}
}
// 1. 封装上报信息
pHeader = (TrackerHeader *)pBuff;
pStatBuff = (TrackerStatReportReqBody*) \
(pBuff + sizeof(TrackerHeader));
long2buff(body_len, pHeader->pkg_len);
pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_REPORT_DISK_USAGE;//【见3.6】
pHeader->status = 0;
for (i=0; i<g_fdfs_store_paths.count; i++)
{
// 读取挂载点磁盘信息
if (statvfs(g_fdfs_store_paths.paths[i], &sbuf) != 0)
{
logError("file: "__FILE__", line: %d, " \
"call statfs fail, errno: %d, error info: %s.",\
__LINE__, errno, STRERROR(errno));
if (pBuff != out_buff)
{
free(pBuff);
}
return errno != 0 ? errno : EACCES;
}
// 填充信息
g_path_space_list[i].total_mb = ((int64_t)(sbuf.f_blocks) * \
sbuf.f_frsize) / FDFS_ONE_MB;
g_path_space_list[i].free_mb = ((int64_t)(sbuf.f_bavail) * \
sbuf.f_frsize) / FDFS_ONE_MB;
long2buff(g_path_space_list[i].total_mb, pStatBuff->sz_total_mb);
long2buff(g_path_space_list[i].free_mb, pStatBuff->sz_free_mb);
pStatBuff++;
}
if (g_store_path_mode == FDFS_STORE_PATH_LOAD_BALANCE)
{// store_path 负载均衡模式
int max_free_mb;
/* find the max free space path */
max_free_mb = 0;
store_path_index = -1;
for (i=0; i<g_fdfs_store_paths.count; i++)
{
if (g_path_space_list[i].free_mb > \
g_avg_storage_reserved_mb \
&& g_path_space_list[i].free_mb > max_free_mb)
{
store_path_index = i;
max_free_mb = g_path_space_list[i].free_mb;
}
}
if (g_store_path_index != store_path_index)
{
g_store_path_index = store_path_index;
}
}
// 2.发送
result = tcpsenddata_nb(pTrackerServer->sock, pBuff, \
total_len, g_fdfs_network_timeout);
if (pBuff != out_buff)
{
free(pBuff);
}
if(result != 0)
{
logError("file: "__FILE__", line: %d, " \
"tracker server %s:%d, send data fail, " \
"errno: %d, error info: %s.", \
__LINE__, pTrackerServer->ip_addr, \
pTrackerServer->port, \
result, STRERROR(result));
return result;
}
// 3.接收反馈。根据变化，触发文件同步sync
return tracker_check_response(pTrackerServer, bServerPortChanged);
}
3. tracker report相关源代码分析
tracker在nio线程[tracker/tracker_nio.c]接收函数client_sock_read接收完后，调用tracker_deal_task [tracker/tracker_service.c]处理任务。
3.1 tracker_deal_storage_join函数
根据tracker_deal_task其中代码：
case TRACKER_PROTO_CMD_STORAGE_JOIN:
result = tracker_deal_storage_join(pTask);
break;
storage中上报磁盘信息tracker_report_df_stat函数发送的命令：TRACKER_PROTO_CMD_STORAGE_JOIN，由tracker_deal_storage_join函数处理。
tracker/tracker_service.c
static int tracker_deal_storage_join(struct fast_task_info *pTask)
{
TrackerStorageJoinBodyResp *pJoinBodyResp;
TrackerStorageJoinBody *pBody;
ConnectionInfo *pTrackerServer;
ConnectionInfo *pTrackerEnd;
char *p;
char *pSeperator;
FDFSStorageJoinBody joinBody;
int result;
TrackerClientInfo *pClientInfo;
char tracker_ip[IP_ADDRESS_SIZE];
// 获取client信息
pClientInfo = (TrackerClientInfo *)pTask->arg;
// 判断包长
if (pTask->length - sizeof(TrackerHeader) < \
sizeof(TrackerStorageJoinBody))
{
logError("file: "__FILE__", line: %d, " \
"cmd: %d, client ip: %s, " \
"package size "PKG_LEN_PRINTF_FORMAT" " \
"is not correct, expect length >= %d.", \
__LINE__, TRACKER_PROTO_CMD_STORAGE_JOIN, \
pTask->client_ip, pTask->length - \
(int)sizeof(TrackerHeader),
(int)sizeof(TrackerStorageJoinBody));
pTask->length = sizeof(TrackerHeader);
return EINVAL;
}
// 把接收到的信息，写入FDFSStorageJoinBody joinBody。
pBody = (TrackerStorageJoinBody *)(pTask->data + sizeof(TrackerHeader));
joinBody.tracker_count = buff2long(pBody->tracker_count);
if (joinBody.tracker_count <= 0 || \
joinBody.tracker_count > FDFS_MAX_TRACKERS)
{
logError("file: "__FILE__", line: %d, " \
"cmd: %d, client ip: %s, " \
"tracker_count: %d is invalid, it <= 0 or > %d", \
__LINE__, TRACKER_PROTO_CMD_STORAGE_JOIN, \
pTask->client_ip, joinBody.tracker_count, \
FDFS_MAX_TRACKERS);
pTask->length = sizeof(TrackerHeader);
return EINVAL;
}
if (pTask->length - sizeof(TrackerHeader) != \
sizeof(TrackerStorageJoinBody) + joinBody.tracker_count *\
FDFS_PROTO_IP_PORT_SIZE)
{
logError("file: "__FILE__", line: %d, " \
"cmd: %d, client ip: %s, " \
"package size "PKG_LEN_PRINTF_FORMAT" " \
"is not correct, expect length %d.", \
__LINE__, TRACKER_PROTO_CMD_STORAGE_JOIN, \
pTask->client_ip, pTask->length - \
(int)sizeof(TrackerHeader),
(int)sizeof(TrackerStorageJoinBody) + \
joinBody.tracker_count * FDFS_PROTO_IP_PORT_SIZE);
pTask->length = sizeof(TrackerHeader);
return EINVAL;
}
joinBody.service_guid[FDFS_SERVICE_GUID_LEN] = '\0';
if ((result = fdfs_validate_service_guid(joinBody.service_guid)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"client ip: %s, null service_guid: %s", \
__LINE__, pTask->client_ip, \
joinBody.service_guid);
pTask->length = sizeof(TrackerHeader);
return result;
}
memcpy(joinBody.group_name, pBody->group_name, FDFS_GROUP_NAME_MAX_LEN);
joinBody.group_name[FDFS_GROUP_NAME_MAX_LEN] = '\0';
if ((result=fdfs_validate_group_name(joinBody.group_name)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"client ip: %s, invalid group_name: %s", \
__LINE__, pTask->client_ip, \
joinBody.group_name);
pTask->length = sizeof(TrackerHeader);
return result;
}
joinBody.storage_port = (int)buff2long(pBody->storage_port);
if (joinBody.storage_port <= 0)
{
logError("file: "__FILE__", line: %d, " \
"client ip: %s, invalid port: %d", \
__LINE__, pTask->client_ip, \
joinBody.storage_port);
pTask->length = sizeof(TrackerHeader);
return EINVAL;
}
joinBody.storage_http_port = (int)buff2long(pBody->storage_http_port);
if (joinBody.storage_http_port < 0)
{
logError("file: "__FILE__", line: %d, " \
"client ip: %s, invalid http port: %d", \
__LINE__, pTask->client_ip, \
joinBody.storage_http_port);
pTask->length = sizeof(TrackerHeader);
return EINVAL;
}
joinBody.store_path_count = (int)buff2long(pBody->store_path_count);
if (joinBody.store_path_count <= 0 || joinBody.store_path_count > 256)
{
logError("file: "__FILE__", line: %d, " \
"client ip: %s, invalid store_path_count: %d", \
__LINE__, pTask->client_ip, \
joinBody.store_path_count);
pTask->length = sizeof(TrackerHeader);
return EINVAL;
}
joinBody.subdir_count_per_path = (int)buff2long( \
pBody->subdir_count_per_path);
if (joinBody.subdir_count_per_path <= 0 || \
joinBody.subdir_count_per_path > 256)
{
logError("file: "__FILE__", line: %d, " \
"client ip: %s, invalid subdir_count_per_path: %d", \
__LINE__, pTask->client_ip, \
joinBody.subdir_count_per_path);
pTask->length = sizeof(TrackerHeader);
return EINVAL;
}
// 写入trackers
p = pTask->data+sizeof(TrackerHeader)+sizeof(TrackerStorageJoinBody);
pTrackerEnd = joinBody.tracker_servers + \
joinBody.tracker_count;
for (pTrackerServer=joinBody.tracker_servers; \
pTrackerServer<pTrackerEnd; pTrackerServer++)
{
* (p + FDFS_PROTO_IP_PORT_SIZE - 1) = '\0';
if ((pSeperator=strchr(p, ':')) == NULL)
{
logError("file: "__FILE__", line: %d, " \
"client ip: %s, invalid tracker server ip " \
"and port: %s", __LINE__, pTask->client_ip, p);
pTask->length = sizeof(TrackerHeader);
return EINVAL;
}
*pSeperator = '\0';
snprintf(pTrackerServer->ip_addr, \
sizeof(pTrackerServer->ip_addr), "%s", p);
pTrackerServer->port = atoi(pSeperator + 1);
pTrackerServer->sock = -1;
p += FDFS_PROTO_IP_PORT_SIZE;
}
joinBody.upload_priority = (int)buff2long(pBody->upload_priority);
joinBody.join_time = (time_t)buff2long(pBody->join_time);
joinBody.up_time = (time_t)buff2long(pBody->up_time);
*(pBody->version + (sizeof(pBody->version) - 1)) = '\0';
*(pBody->domain_name + (sizeof(pBody->domain_name) - 1)) = '\0';
strcpy(joinBody.version, pBody->version);
strcpy(joinBody.domain_name, pBody->domain_name);
joinBody.init_flag = pBody->init_flag;
joinBody.status = pBody->status;
getSockIpaddr(pTask->event.fd, \
tracker_ip, IP_ADDRESS_SIZE);
insert_into_local_host_ip(tracker_ip);
// 根据joinBody更新或者新增storage和组
result = tracker_mem_add_group_and_storage(pClientInfo, \
pTask->client_ip, &joinBody, true);
if (result != 0)
{
pTask->length = sizeof(TrackerHeader);
return result;
}
// 打包返回信息
pJoinBodyResp = (TrackerStorageJoinBodyResp *)(pTask->data + \
sizeof(TrackerHeader));
memset(pJoinBodyResp, 0, sizeof(TrackerStorageJoinBodyResp));
if (pClientInfo->pStorage->psync_src_server != NULL)
{
strcpy(pJoinBodyResp->src_id, \
pClientInfo->pStorage->psync_src_server->id);
}
pTask->length = sizeof(TrackerHeader) + \
sizeof(TrackerStorageJoinBodyResp);
return 0;
}
3.2 tracker_deal_storage_sync_notify函数
根据tracker_deal_task其中代码：
case TRACKER_PROTO_CMD_STORAGE_SYNC_NOTIFY:
result = tracker_deal_storage_sync_notify(pTask);
break;
storage中上报更新源ip和时间戳函数tracker_sync_notify函数发送的命令：TRACKER_PROTO_CMD_STORAGE_SYNC_NOTIFY，由tracker_deal_storage_sync_notify函数处理。
tracker/tracker_service.c
static int tracker_deal_storage_sync_notify(struct fast_task_info *pTask)
{
TrackerStorageSyncReqBody *pBody;
char sync_src_id[FDFS_STORAGE_ID_MAX_SIZE];
bool bSaveStorages;
TrackerClientInfo *pClientInfo;
// 1.获取client信息
pClientInfo = (TrackerClientInfo *)pTask->arg;
// 2.判断包长
if (pTask->length - sizeof(TrackerHeader) != \
sizeof(TrackerStorageSyncReqBody))
{
logError("file: "__FILE__", line: %d, " \
"cmd: %d, client ip: %s, package size " \
PKG_LEN_PRINTF_FORMAT" is not correct, " \
"expect length: %d", __LINE__, \
TRACKER_PROTO_CMD_STORAGE_SYNC_NOTIFY, \
pTask->client_ip, pTask->length - \
(int)sizeof(TrackerHeader),
(int)sizeof(TrackerStorageSyncReqBody));
pTask->length = sizeof(TrackerHeader);
return EINVAL;
}
// 获取消息体
pBody=(TrackerStorageSyncReqBody *)(pTask->data+sizeof(TrackerHeader));
if (*(pBody->src_id) == '\0')
{
if (pClientInfo->pStorage->status == FDFS_STORAGE_STATUS_INIT || \
pClientInfo->pStorage->status == FDFS_STORAGE_STATUS_WAIT_SYNC || \
pClientInfo->pStorage->status == FDFS_STORAGE_STATUS_SYNCING)
{//更新storage运行状态，并保存文件
pClientInfo->pStorage->status = FDFS_STORAGE_STATUS_ONLINE;
pClientInfo->pGroup->chg_count++;
tracker_save_storages();
}
pTask->length = sizeof(TrackerHeader);
return 0;
}
bSaveStorages = false;
if (pClientInfo->pStorage->status == FDFS_STORAGE_STATUS_INIT)
{
pClientInfo->pStorage->status = FDFS_STORAGE_STATUS_WAIT_SYNC;
pClientInfo->pGroup->chg_count++;
bSaveStorages = true;
}
if (pClientInfo->pStorage->psync_src_server == NULL)
{// 如果源同步服务为空，就根据src_id获取源同步服务
memcpy(sync_src_id, pBody->src_id, \
FDFS_STORAGE_ID_MAX_SIZE);
sync_src_id[FDFS_STORAGE_ID_MAX_SIZE - 1] = '\0';
// 根据src_id获取源同步服务
pClientInfo->pStorage->psync_src_server = \
tracker_mem_get_storage(pClientInfo->pGroup, \
sync_src_id);
if (pClientInfo->pStorage->psync_src_server == NULL)
{
logError("file: "__FILE__", line: %d, " \
"client ip: %s, " \
"sync src server: %s not exists", \
__LINE__, pTask->client_ip, \
sync_src_id);
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}
if (pClientInfo->pStorage->psync_src_server->status == \
FDFS_STORAGE_STATUS_DELETED)
{
logError("file: "__FILE__", line: %d, " \
"client ip: %s, " \
"sync src server: %s already be deleted", \
__LINE__, pTask->client_ip, \
sync_src_id);
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}
if (pClientInfo->pStorage->psync_src_server->status == \
FDFS_STORAGE_STATUS_IP_CHANGED)
{
logError("file: "__FILE__", line: %d, " \
"client ip: %s, the ip address of " \
"the sync src server: %s changed", \
__LINE__, pTask->client_ip, \
sync_src_id);
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}
pClientInfo->pStorage->sync_until_timestamp = \
(int)buff2long(pBody->until_timestamp);
bSaveStorages = true;
}
if (bSaveStorages)
{
// 把状态保存到文件
tracker_save_storages();
}
pTask->length = sizeof(TrackerHeader);
return 0;
}
3.3 tracker_deal_storage_sync_dest_req函数
根据tracker_deal_task其中代码：
case TRACKER_PROTO_CMD_STORAGE_SYNC_DEST_REQ:
TRACKER_CHECK_LOGINED(pTask)
result = tracker_deal_storage_sync_dest_req(pTask);
break;
storage中请求同步信息tracker_sync_dest_req函数发送的命令：TRACKER_PROTO_CMD_STORAGE_SYNC_DEST_REQ，由tracker_deal_storage_sync_dest_req函数处理。
tracker/tracker_service.c
static int tracker_deal_storage_sync_dest_req(struct fast_task_info *pTask)
{
TrackerStorageSyncReqBody *pBody;
FDFSStorageDetail *pSrcStorage;
FDFSStorageDetail **ppServer;
FDFSStorageDetail **ppServerEnd;
int sync_until_timestamp;
int source_count;
TrackerClientInfo *pClientInfo;
// 1.获取client信息
pClientInfo = (TrackerClientInfo *)pTask->arg;
// 2.获同步时间戳
pSrcStorage = NULL;
sync_until_timestamp = (int)g_current_time;
// 3.获取更新源 src_id
do
{
if (pTask->length - sizeof(TrackerHeader) != 0)
{
logError("file: "__FILE__", line: %d, " \
"cmd=%d, client ip: %s, package size " \
PKG_LEN_PRINTF_FORMAT" is not correct, " \
"expect length: 0", \
__LINE__, TRACKER_PROTO_CMD_STORAGE_SYNC_DEST_REQ, \
pTask->client_ip, pTask->length - \
(int)sizeof(TrackerHeader));
pTask->length = sizeof(TrackerHeader);
return EINVAL;
}
// 如果组内仅不多于一个storage，返回
if (pClientInfo->pGroup->count <= 1)
{
break;
}
source_count = 0;
ppServerEnd = pClientInfo->pGroup->all_servers + \
pClientInfo->pGroup->count;
for (ppServer=pClientInfo->pGroup->all_servers; \
ppServer<ppServerEnd; ppServer++)
{
// 如果是自己，跳过
if (strcmp((*ppServer)->id, \
pClientInfo->pStorage->id) == 0)
{
continue;
}
// 统计可能更新源的个数
if ((*ppServer)->status == FDFS_STORAGE_STATUS_OFFLINE
|| (*ppServer)->status == FDFS_STORAGE_STATUS_ONLINE
|| (*ppServer)->status == FDFS_STORAGE_STATUS_ACTIVE)
{
source_count++;
}
}
// 无更新源，返回
if (source_count == 0)
{
break;
}
// 在active的组内storages抽取第一个作为该storage的src_storage
pSrcStorage = tracker_get_group_sync_src_server( \
pClientInfo->pGroup, pClientInfo->pStorage);
if (pSrcStorage == NULL)
{
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}
// 4.封装返回内容，src_id和sync_until_timestamp
pBody=(TrackerStorageSyncReqBody *)(pTask->data+sizeof(TrackerHeader));
strcpy(pBody->src_id, pSrcStorage->id);
long2buff(sync_until_timestamp, pBody->until_timestamp);
} while (0);
if (pSrcStorage == NULL)
{
pClientInfo->pStorage->status = \
FDFS_STORAGE_STATUS_ONLINE;
pClientInfo->pGroup->chg_count++;
tracker_save_storages();
pTask->length = sizeof(TrackerHeader);
return 0;
}
// 更新tracker中保存的storage信息
pClientInfo->pStorage->psync_src_server = pSrcStorage;
pClientInfo->pStorage->sync_until_timestamp = sync_until_timestamp;
pClientInfo->pStorage->status = FDFS_STORAGE_STATUS_WAIT_SYNC;
pClientInfo->pGroup->chg_count++;
// 把storage信息保存到文件中
tracker_save_storages();
pTask->length = sizeof(TrackerHeader)+sizeof(TrackerStorageSyncReqBody);
return 0;
}
3.4 tracker_deal_storage_beat函数
根据tracker_deal_task其中代码：
case TRACKER_PROTO_CMD_STORAGE_BEAT:
TRACKER_CHECK_LOGINED(pTask)
result = tracker_deal_storage_beat(pTask);
break;
storage中storage心跳包tracker_heart_beat函数发送的命令：TRACKER_PROTO_CMD_STORAGE_BEAT，由tracker_deal_storage_beat函数处理。
tracker/tracker_service.c
static int tracker_deal_storage_beat(struct fast_task_info *pTask)
{
int nPkgLen;
int status;
FDFSStorageStatBuff *pStatBuff;
FDFSStorageStat *pStat;
TrackerClientInfo *pClientInfo;
// 1.获取client信息
pClientInfo = (TrackerClientInfo *)pTask->arg;
do
{
// 2. 判断包长
nPkgLen = pTask->length - sizeof(TrackerHeader);
if (nPkgLen == 0)
{
status = 0;
break;
}
if (nPkgLen != sizeof(FDFSStorageStatBuff))
{
logError("file: "__FILE__", line: %d, " \
"cmd=%d, client ip: %s, package size " \
PKG_LEN_PRINTF_FORMAT" is not correct, " \
"expect length: 0 or %d", __LINE__, \
TRACKER_PROTO_CMD_STORAGE_BEAT, \
pTask->client_ip, nPkgLen,
(int)sizeof(FDFSStorageStatBuff));
status = EINVAL;
break;
}
// 3.把上报的信息更新到tracker记录的storage中
pStatBuff = (FDFSStorageStatBuff *)(pTask->data + \
sizeof(TrackerHeader));
pStat = &(pClientInfo->pStorage->stat);
pStat->connection.alloc_count = \
buff2int(pStatBuff->connection.sz_alloc_count);
pStat->connection.current_count = \
buff2int(pStatBuff->connection.sz_current_count);
pStat->connection.max_count = \
buff2int(pStatBuff->connection.sz_max_count);
pStat->total_upload_count = \
buff2long(pStatBuff->sz_total_upload_count);
pStat->success_upload_count = \
buff2long(pStatBuff->sz_success_upload_count);
pStat->total_append_count = \
buff2long(pStatBuff->sz_total_append_count);
pStat->success_append_count = \
buff2long(pStatBuff->sz_success_append_count);
pStat->total_modify_count = \
buff2long(pStatBuff->sz_total_modify_count);
pStat->success_modify_count = \
buff2long(pStatBuff->sz_success_modify_count);
pStat->total_truncate_count = \
buff2long(pStatBuff->sz_total_truncate_count);
pStat->success_truncate_count = \
buff2long(pStatBuff->sz_success_truncate_count);
pStat->total_download_count = \
buff2long(pStatBuff->sz_total_download_count);
pStat->success_download_count = \
buff2long(pStatBuff->sz_success_download_count);
pStat->total_set_meta_count = \
buff2long(pStatBuff->sz_total_set_meta_count);
pStat->success_set_meta_count = \
buff2long(pStatBuff->sz_success_set_meta_count);
pStat->total_delete_count = \
buff2long(pStatBuff->sz_total_delete_count);
pStat->success_delete_count = \
buff2long(pStatBuff->sz_success_delete_count);
pStat->total_get_meta_count = \
buff2long(pStatBuff->sz_total_get_meta_count);
pStat->success_get_meta_count = \
buff2long(pStatBuff->sz_success_get_meta_count);
pStat->last_source_update = \
buff2long(pStatBuff->sz_last_source_update);
pStat->last_sync_update = \
buff2long(pStatBuff->sz_last_sync_update);
pStat->total_create_link_count = \
buff2long(pStatBuff->sz_total_create_link_count);
pStat->success_create_link_count = \
buff2long(pStatBuff->sz_success_create_link_count);
pStat->total_delete_link_count = \
buff2long(pStatBuff->sz_total_delete_link_count);
pStat->success_delete_link_count = \
buff2long(pStatBuff->sz_success_delete_link_count);
pStat->total_upload_bytes = \
buff2long(pStatBuff->sz_total_upload_bytes);
pStat->success_upload_bytes = \
buff2long(pStatBuff->sz_success_upload_bytes);
pStat->total_append_bytes = \
buff2long(pStatBuff->sz_total_append_bytes);
pStat->success_append_bytes = \
buff2long(pStatBuff->sz_success_append_bytes);
pStat->total_modify_bytes = \
buff2long(pStatBuff->sz_total_modify_bytes);
pStat->success_modify_bytes = \
buff2long(pStatBuff->sz_success_modify_bytes);
pStat->total_download_bytes = \
buff2long(pStatBuff->sz_total_download_bytes);
pStat->success_download_bytes = \
buff2long(pStatBuff->sz_success_download_bytes);
pStat->total_sync_in_bytes = \
buff2long(pStatBuff->sz_total_sync_in_bytes);
pStat->success_sync_in_bytes = \
buff2long(pStatBuff->sz_success_sync_in_bytes);
pStat->total_sync_out_bytes = \
buff2long(pStatBuff->sz_total_sync_out_bytes);
pStat->success_sync_out_bytes = \
buff2long(pStatBuff->sz_success_sync_out_bytes);
pStat->total_file_open_count = \
buff2long(pStatBuff->sz_total_file_open_count);
pStat->success_file_open_count = \
buff2long(pStatBuff->sz_success_file_open_count);
pStat->total_file_read_count = \
buff2long(pStatBuff->sz_total_file_read_count);
pStat->success_file_read_count = \
buff2long(pStatBuff->sz_success_file_read_count);
pStat->total_file_write_count = \
buff2long(pStatBuff->sz_total_file_write_count);
pStat->success_file_write_count = \
buff2long(pStatBuff->sz_success_file_write_count);
// 4.把更新内容写入文件
if (++g_storage_stat_chg_count % TRACKER_SYNC_TO_FILE_FREQ == 0)
{
status = tracker_save_storages();
}
else
{
status = 0;
}
//printf("g_storage_stat_chg_count=%d\n", g_storage_stat_chg_count);
} while (0);
if (status == 0)
{
// 5.更新该storage的状态为active，并且更新最新的心跳时间
tracker_mem_active_store_server(pClientInfo->pGroup, \
pClientInfo->pStorage);
pClientInfo->pStorage->stat.last_heart_beat_time = g_current_time;
}
// 6.返回同步信息
//printf("deal heart beat, status=%d\n", status);
return tracker_check_and_sync(pTask, status);
}
3.5 tracker_deal_storage_sync_report函数
根据tracker_deal_task其中代码：
case TRACKER_PROTO_CMD_STORAGE_SYNC_REPORT:
TRACKER_CHECK_LOGINED(pTask)
result = tracker_deal_storage_sync_report(pTask);
break;
storage中上报与同组storage同步的时间戳tracker_report_sync_timestamp函数发送的命令：TRACKER_PROTO_CMD_STORAGE_SYNC_REPORT，由tracker_deal_storage_sync_report函数处理。
tracker/tracker_service.c
static int tracker_deal_storage_sync_report(struct fast_task_info *pTask)
{
char *p;
char *pEnd;
char *src_id;
int status;
int sync_timestamp;
int src_index;
int dest_index;
int nPkgLen;
FDFSStorageDetail *pSrcStorage;
TrackerClientInfo *pClientInfo;
// 1.获取client信息
pClientInfo = (TrackerClientInfo *)pTask->arg;
// 2.判断包长
nPkgLen = pTask->length - sizeof(TrackerHeader);
if (nPkgLen <= 0 || nPkgLen % (FDFS_STORAGE_ID_MAX_SIZE + 4) != 0)
{
logError("file: "__FILE__", line: %d, " \
"cmd=%d, client ip: %s, package size " \
PKG_LEN_PRINTF_FORMAT" is not correct", \
__LINE__, TRACKER_PROTO_CMD_STORAGE_SYNC_REPORT, \
pTask->client_ip, nPkgLen);
pTask->length = sizeof(TrackerHeader);
return EINVAL;
}
// 3. 更新最新同步时间戳last_synced_timestamp
do
{
// 获此上报storage的在组内的index
dest_index = tracker_mem_get_storage_index(pClientInfo->pGroup,
pClientInfo->pStorage);
if (dest_index < 0 || dest_index >= pClientInfo->pGroup->count)
{
status = 0;
break;
}
// 如果是轮询上传
if (g_groups.store_server == FDFS_STORE_SERVER_ROUND_ROBIN)
{ // pxxian: ROUND ROBIN
int min_synced_timestamp;
min_synced_timestamp = 0;
pEnd = pTask->data + pTask->length;
for (p=pTask->data + sizeof(TrackerHeader); p<pEnd; \
p += (FDFS_STORAGE_ID_MAX_SIZE + 4))
{
// 读取上报信息src_id和sync_timestamp
sync_timestamp = buff2int(p + FDFS_STORAGE_ID_MAX_SIZE);
if (sync_timestamp <= 0)
{
continue;
}
src_id = p;
*(src_id + (FDFS_STORAGE_ID_MAX_SIZE - 1)) = '\0';
// 获取src_id指向的storage
pSrcStorage = tracker_mem_get_storage( \
pClientInfo->pGroup, src_id);
if (pSrcStorage == NULL)
{
continue;
}
if (pSrcStorage->status != FDFS_STORAGE_STATUS_ACTIVE)
{
continue;
}
// 获取src_id的storage的组内index
src_index = tracker_mem_get_storage_index( \
pClientInfo->pGroup, pSrcStorage);
if (src_index == dest_index || src_index < 0 || \
src_index >= pClientInfo->pGroup->count)
{
continue;
}
// 更新同步时间戳sync_timestamp
pClientInfo->pGroup->last_sync_timestamps \
[src_index][dest_index] = sync_timestamp;
// 查找最早的“最后更新时间戳”。
if (min_synced_timestamp == 0)
{
min_synced_timestamp = sync_timestamp;
}
else if (sync_timestamp < min_synced_timestamp)
{
min_synced_timestamp = sync_timestamp;
}
}
// 把最早的“最后更新时间戳”作为“最后的更新时间戳”。这意味着该storage保存了组内早于这个时间的所有文件。
if (min_synced_timestamp > 0)
{
pClientInfo->pStorage->stat.last_synced_timestamp = \
min_synced_timestamp;
}
}
else
{ // 在指定源、权重上传模式
int max_synced_timestamp;
max_synced_timestamp = pClientInfo->pStorage->stat.\
last_synced_timestamp;
pEnd = pTask->data + pTask->length;
for (p=pTask->data + sizeof(TrackerHeader); p<pEnd; \
p += (FDFS_STORAGE_ID_MAX_SIZE + 4))
{
// 读取上报信息src_id和sync_timestamp
sync_timestamp = buff2int(p + FDFS_STORAGE_ID_MAX_SIZE);
if (sync_timestamp <= 0)
{
continue;
}
src_id = p;
*(src_id + (FDFS_STORAGE_ID_MAX_SIZE - 1)) = '\0';
// 获取src_id指向的storage
pSrcStorage = tracker_mem_get_storage( \
pClientInfo->pGroup, src_id);
if (pSrcStorage == NULL)
{
continue;
}
if (pSrcStorage->status != FDFS_STORAGE_STATUS_ACTIVE)
{
continue;
}
// 获取src_id的storage的组内index
src_index = tracker_mem_get_storage_index( \
pClientInfo->pGroup, pSrcStorage);
if (src_index == dest_index || src_index < 0 || \
src_index >= pClientInfo->pGroup->count)
{
continue;
}
// 更新同步时间戳sync_timestamp
pClientInfo->pGroup->last_sync_timestamps \
[src_index][dest_index] = sync_timestamp;
// 查找最后同步时间
if (sync_timestamp > max_synced_timestamp)
{
max_synced_timestamp = sync_timestamp;
}
}
// 更新最后同步时间
pClientInfo->pStorage->stat.last_synced_timestamp = \
max_synced_timestamp;
}
// 信息保存到文件
if (++g_storage_sync_time_chg_count % \
TRACKER_SYNC_TO_FILE_FREQ == 0)
{
status = tracker_save_sync_timestamps();
}
else
{
status = 0;
}
} while (0);
// 返回同步信息
return tracker_check_and_sync(pTask, status);
}
3.6 tracker_deal_storage_df_report函数
根据tracker_deal_task其中代码：
case TRACKER_PROTO_CMD_STORAGE_REPORT_DISK_USAGE:
TRACKER_CHECK_LOGINED(pTask)
result = tracker_deal_storage_df_report(pTask);
break;
storage中上报磁盘信息tracker_report_df_stat函数发送的命令：TRACKER_PROTO_CMD_STORAGE_REPORT_DISK_USAGE，由tracker_deal_storage_df_report函数处理。
tracker/tracker_service.c
static int tracker_deal_storage_df_report(struct fast_task_info *pTask)
{
int nPkgLen;
int i;
TrackerStatReportReqBody *pStatBuff;
int64_t *path_total_mbs;
int64_t *path_free_mbs;
int64_t old_free_mb;
TrackerClientInfo *pClientInfo;
// 获取client信息
pClientInfo = (TrackerClientInfo *)pTask->arg;
if (pClientInfo->pGroup == NULL || pClientInfo->pStorage == NULL)
{
logError("file: "__FILE__", line: %d, " \
"cmd=%d, client ip: %s, not join in!", \
__LINE__, TRACKER_PROTO_CMD_STORAGE_REPORT_DISK_USAGE, \
pTask->client_ip);
pTask->length = sizeof(TrackerHeader);
return EINVAL;
}
// 判断包长度正确性
nPkgLen = pTask->length - sizeof(TrackerHeader);
if (nPkgLen != sizeof(TrackerStatReportReqBody) * \
pClientInfo->pGroup->store_path_count)
{
logError("file: "__FILE__", line: %d, " \
"cmd=%d, client ip: %s, package size " \
PKG_LEN_PRINTF_FORMAT" is not correct, " \
"expect length: %d", __LINE__, \
TRACKER_PROTO_CMD_STORAGE_REPORT_DISK_USAGE, \
pTask->client_ip, nPkgLen, \
(int)sizeof(TrackerStatReportReqBody) * \
pClientInfo->pGroup->store_path_count);
pTask->length = sizeof(TrackerHeader);
return EINVAL;
}
// 更新组存储、挂载点存储信息，剩余空间和总空间
old_free_mb = pClientInfo->pStorage->free_mb;
path_total_mbs = pClientInfo->pStorage->path_total_mbs;
path_free_mbs = pClientInfo->pStorage->path_free_mbs;
pClientInfo->pStorage->total_mb = 0;
pClientInfo->pStorage->free_mb = 0;
pStatBuff = (TrackerStatReportReqBody *)(pTask->data + sizeof(TrackerHeader));
for (i=0; i<pClientInfo->pGroup->store_path_count; i++)
{
// pxxian: renew the store_path memory message
path_total_mbs[i] = buff2long(pStatBuff->sz_total_mb);
path_free_mbs[i] = buff2long(pStatBuff->sz_free_mb);
pClientInfo->pStorage->total_mb += path_total_mbs[i];
pClientInfo->pStorage->free_mb += path_free_mbs[i];
if (g_groups.store_path == FDFS_STORE_PATH_LOAD_BALANCE
&& path_free_mbs[i] > path_free_mbs[ \
pClientInfo->pStorage->current_write_path])
{
pClientInfo->pStorage->current_write_path = i;
}
pStatBuff++;
}
// 更新组剩余空间，为组内各storage中最小的
if ((pClientInfo->pGroup->free_mb == 0) ||
(pClientInfo->pStorage->free_mb < pClientInfo->pGroup->free_mb))
{
pClientInfo->pGroup->total_mb = pClientInfo->pStorage->total_mb;
pClientInfo->pGroup->free_mb = pClientInfo->pStorage->free_mb;
}
else if (pClientInfo->pStorage->free_mb > old_free_mb)
{
tracker_find_min_free_space(pClientInfo->pGroup);
}
// 查找剩余空间最大的组，把写入指向标志更新指向该组
tracker_find_max_free_space_group();
/*
//logInfo("storage: %s:%d, total_mb=%dMB, free_mb=%dMB\n", \
pClientInfo->pStorage->ip_addr, \
pClientInfo->pGroup->storage_port, \
pClientInfo->pStorage->total_mb, \
pClientInfo->pStorage->free_mb);
*/
// 把该storage标志为active
tracker_mem_active_store_server(pClientInfo->pGroup, \
pClientInfo->pStorage);
// 返回同步信息
return tracker_check_and_sync(pTask, 0);
}
368
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
