关于MYSQL 5.6 super_read_only和Event Scheduler冲突导致启动失败
版本percona-server-5.6.22-72.0，这个版本肯定有这个问题
这个问题出现在线上我们将库设置为super_read_only后启动报错，整个MYSQLD CRASH掉
2017-04-23 01:15:46 22577 [ERROR] Event Scheduler: Failed to open table mysql.event
2017-04-23 01:15:46 22577 [ERROR] Event Scheduler: Error while loading from disk.
2017-04-23 01:15:46 22577 [Note] Event Scheduler: Purging the queue. 0 events
2017-04-23 01:15:46 22577 [ERROR] Aborting
在5.6.25以及其他5.7的版本上并没有这个问题。网上说可能是BUG但是没有说出具体原因，也有
帖子说和super_read_only 有关，确实关闭super_read_only就不会再出问题，
但是为了找到问题，我想挖一挖代码如下：
首先要找到报错源头，首先找到Event Scheduler: Error while loading from disk.的位置如下：
这段代码出现在
events.cc的Events::init(my_bool opt_noacl_or_bootstrap) 方法下
if (event_queue->init_queue(thd) || load_events_from_db(thd) ||
(opt_event_scheduler == EVENTS_ON && scheduler->start(&err_no)))
{
sql_print_error("Event Scheduler: Error while loading from disk."); --这里
res= TRUE; /* fatal error: request unireg_abort */
goto end;
}
这里3个条件都可能引起这个错误：
1、初始化队列失败，函数接口event_queue->init_queue
2、加载event数据失败，函数接口load_events_from_db
3、参数event_scheduler设置是否为ON，并且scheduler->start调用失败
而错误 Event Scheduler: Failed to open table mysql.event 正是load_events_from_db(thd)这个
方法报出来的，查看其部分代码
/*
NOTE: even if we run in read-only mode, we should be able to lock the
mysql.event table for writing. In order to achieve this, we should call
mysql_lock_tables() under the super user.
Same goes for transaction access mode.
Temporarily reset it to read-write.
*/--这里源码也有不算清楚的解释
saved_master_access= thd->security_ctx->master_access;
thd->security_ctx->master_access |= SUPER_ACL;
bool save_tx_read_only= thd->tx_read_only;
thd->tx_read_only= false;
ret= db_repository->open_event_table(thd, TL_WRITE, &table); --这里的返回值进行判断
thd->tx_read_only= save_tx_read_only;
thd->security_ctx->master_access= saved_master_access;
if (ret)
{
sql_print_error("Event Scheduler: Failed to open table mysql.event"); ---这里
DBUG_RETURN(TRUE);
}
我们可以看到他的是通过调用db_repository->open_event_table(thd, TL_WRITE, &table)来
接收其返回值如果为true则报错。接下来看open_event_table
bool
Event_db_repository::open_event_table(THD *thd, enum thr_lock_type lock_type,
TABLE **table)
{
TABLE_LIST tables;
DBUG_ENTER("Event_db_repository::open_event_table");
tables.init_one_table("mysql", 5, "event", 5, "event", lock_type);
if (open_and_lock_tables(thd, &tables, FALSE, MYSQL_LOCK_IGNORE_TIMEOUT))
DBUG_RETURN(TRUE);
*table= tables.table;
tables.table->use_all_columns();
if (table_intact.check(*table, &event_table_def))
{
close_thread_tables(thd);
my_error(ER_EVENT_OPEN_TABLE_FAILED, MYF(0));
DBUG_RETURN(TRUE);
}
DBUG_RETURN(FALSE);
}
大概这个函数会判断
1、event表是否可以lock
2、event表是否损坏
最终会调用lock_tables-->mysql_lock_tables，如果mysql_lock_locks返回一个
NULL指针则报错如下：
DEBUG_SYNC(thd, "before_lock_tables_takes_lock");
if (! (thd->lock= mysql_lock_tables(thd, start, (uint) (ptr - start), flags))) --如果mysql_lock_tables返回一个NULL给thd->lock，产生异常
DBUG_RETURN(TRUE);
DEBUG_SYNC(thd, "after_lock_tables_takes_lock");
接下来mysql_lock_tables 又调用lock_tables_check函数进行table
lock的检查，如果lock_tables_check函数返回一个大于0的值则异常
那么调用流程清晰了
Events::init-->Events::load_events_from_db-->open_event_table
-->open_and_lock_tables -->lock_tables-->mysql_lock_tables-->
lock_tables_check
最终我们分析掉这个版本的BUG由于lock_tables_check函数检查返回异常
而导致这个错误。
函数调用栈如下:
#0 mysql_lock_tables (thd=0x1c0b5e0, tables=0x1b62ca0, count=1, flags=2048) at /home/percona-server-5.6.22-72.0/sql/lock.cc:296
#1 0x00000000007910c9 in lock_tables (thd=0x1c0b5e0, tables=0x7fffffffdae0, count=1, flags=2048) at /home/percona-server-5.6.22-72.0/sql/sql_base.cc:6125
#2 0x000000000079086f in open_and_lock_tables (thd=0x1c0b5e0, tables=0x7fffffffdae0, derived=false, flags=2048, prelocking_strategy=0x7fffffffda90)
at /home/percona-server-5.6.22-72.0/sql/sql_base.cc:5889
#3 0x0000000000781ed2 in open_and_lock_tables (thd=0x1c0b5e0, tables=0x7fffffffdae0, derived=false, flags=2048)
at /home/percona-server-5.6.22-72.0/sql/sql_base.h:477
#4 0x0000000000a26d20 in Event_db_repository::check_system_tables (thd=0x1c0b5e0) at /home/percona-server-5.6.22-72.0/sql/event_db_repository.cc:1202
#5 0x00000000008ff5fb in Events::init (opt_noacl_or_bootstrap=0 '\000') at /home/percona-server-5.6.22-72.0/sql/events.cc:858
#6 0x000000000063e21d in mysqld_main (argc=83, argv=0x18f4c58) at /home/percona-server-5.6.22-72.0/sql/mysqld.cc:5784
#7 0x0000000000632634 in main (argc=11, argv=0x7fffffffe398) at /home/percona-server-5.6.22-72.0/sql/main.cc:25
跟踪lock_tables_check函数发现如下
198 if (!(flags & MYSQL_LOCK_IGNORE_GLOBAL_READ_ONLY) && !t->s->tmp_table)
(gdb) n
200 if (t->reginfo.lock_type >= TL_WRITE_ALLOW_WRITE &&
(gdb) n
204 opt_super_readonly ? "--read-only (super)" : "--read-only");
(gdb) n
205 DBUG_RETURN(1);
上面由于这段代码：
200 if (t->reginfo.lock_type >= TL_WRITE_ALLOW_WRITE &&
201 enforce_ro && opt_readonly && !thd->slave_thread)
202 {
203 my_error(ER_OPTION_PREVENTS_STATEMENT, MYF(0),
204 opt_super_readonly ? "--read-only (super)" : "--read-only");
205 DBUG_RETURN(1);
206 }
if (t->reginfo.lock_type >= TL_WRITE_ALLOW_WRITE && enforce_ro && opt_readonly && !thd->slave_thread)
条件满足返回了 DBUG_RETURN(1); 造成整个报错
而这里
enforce_ro 起到了关键作用，也是问题的根源，相信和这里有关
if (!opt_super_readonly)
enforce_ro= !(thd->security_ctx->master_access & SUPER_ACL);
(gdb) p t->alias
$12 = 0x1c5b4d0 "event"
(gdb) list
200 if (t->reginfo.lock_type >= TL_WRITE_ALLOW_WRITE &&
201 enforce_ro && opt_readonly && !thd->slave_thread)
202 {
203 my_error(ER_OPTION_PREVENTS_STATEMENT, MYF(0),
204 opt_super_readonly ? "--read-only (super)" : "--read-only");
205 DBUG_RETURN(1);
206 }
207 }
208 }
209
(gdb) p enforce_ro
$13 = true
(gdb) p t->alias
$15 = 0x1c5b4d0 "event"
(gdb) p thd->security_ctx->master_access
$16 = 32768
我们研究下
lock_tables_check(THD *thd, TABLE **tables, uint count, uint flags)
{
uint system_count= 0, i= 0;
bool enforce_ro= true;
/*
Identifies if the executed sql command can updated either a log
or rpl info table.
*/
bool log_table_write_query= false;
DBUG_ENTER("lock_tables_check");
if (!opt_super_readonly)
enforce_ro= !(thd->security_ctx->master_access & SUPER_ACL);
后面省略
这句问题的根源，默认enforce_ro= true;
如果opt_super_readonly开启则
enforce_ro= !(thd->security_ctx->master_access & SUPER_ACL);不运行
那么enforce_ro= true
如果opt_super_readonly不开启则
enforce_ro= !(thd->security_ctx->master_access & SUPER_ACL);运行
SUPER_ACL是一个宏#define SUPER_ACL(1L << 15)
当做4字节INT的话,及
1000 0000 0000 0000
而thd->security_ctx->master_access是32768及
1000 0000 0000 0000
显然他们按位与得到是1 及ture，然后!true
所以enforce_ro= false;
如果为flase则
200 if (t->reginfo.lock_type >= TL_WRITE_ALLOW_WRITE &&
201 enforce_ro && opt_readonly && !thd->slave_thread)
202 {
203 my_error(ER_OPTION_PREVENTS_STATEMENT, MYF(0),
204 opt_super_readonly ? "--read-only (super)" : "--read-only");
205 DBUG_RETURN(1);
206 }
不会执行，则不会再次报错
所以解决这个问题或者说BUG，就是设置如果opt_super_readonly不开启，
就是不要设置super_read_only权限。
在5.7.14版本中，我粗略查看lock_tables_check代码，有改动。5.7.14没遇到这个 问题
if (t->reginfo.lock_type >= TL_WRITE_ALLOW_WRITE &&
check_readonly(thd, true))
DBUG_RETURN(1);
可以看到enforce_ro已经不再作为判断的标准
而5.6.22这个版本确实有这个问题，但是这个问题不是每个版本都有。如果遇到可以参考。
附带5.6.22，5.7.14代码：
5.6.22
点击(此处)折叠或打开
static int
lock_tables_check(THD *thd, TABLE **tables, uint count, uint flags)
{
uint system_count= 0, i= 0;
bool enforce_ro= true;
/*
Identifies if the executed sql command can updated either a log
or rpl info table.
*/
bool log_table_write_query= false;
DBUG_ENTER("lock_tables_check");
if (!opt_super_readonly)
enforce_ro= !(thd->security_ctx->master_access & SUPER_ACL);
log_table_write_query=
is_log_table_write_query(thd->lex->sql_command);
for (i=0 ; i
{
TABLE *t= tables[i];
/* Protect against 'fake' partially initialized TABLE_SHARE */
DBUG_ASSERT(t->s->table_category != TABLE_UNKNOWN_CATEGORY);
/*
Table I/O to performance schema tables is performed
only internally by the server implementation.
When a user is requesting a lock, the following
constraints are enforced:
*/
if (t->s->table_category == TABLE_CATEGORY_LOG &&
(flags & MYSQL_LOCK_LOG_TABLE) == 0 &&
!log_table_write_query)
{
/*
A user should not be able to prevent writes,
or hold any type of lock in a session,
since this would be a DOS attack.
*/
if (t->reginfo.lock_type >= TL_READ_NO_INSERT ||
thd->lex->sql_command == SQLCOM_LOCK_TABLES)
{
my_error(ER_CANT_LOCK_LOG_TABLE, MYF(0));
DBUG_RETURN(1);
}
}
if (t->reginfo.lock_type >= TL_WRITE_ALLOW_WRITE)
{
if (t->s->table_category == TABLE_CATEGORY_SYSTEM)
system_count++;
if (t->db_stat & HA_READ_ONLY)
{
my_error(ER_OPEN_AS_READONLY, MYF(0), t->alias);
DBUG_RETURN(1);
}
}
/*
If we are going to lock a non-temporary table we must own metadata
lock of appropriate type on it (I.e. for table to be locked for
write we must own metadata lock of MDL_SHARED_WRITE or stronger
type. For table to be locked for read we must own metadata lock
of MDL_SHARED_READ or stronger type).
The only exception are HANDLER statements which are allowed to
lock table for read while having only MDL_SHARED lock on it.
*/
DBUG_ASSERT(t->s->tmp_table ||
thd->mdl_context.is_lock_owner(MDL_key::TABLE,
t->s->db.str, t->s->table_name.str,
t->reginfo.lock_type >= TL_WRITE_ALLOW_WRITE ?
MDL_SHARED_WRITE : MDL_SHARED_READ) ||
(t->open_by_handler &&
thd->mdl_context.is_lock_owner(MDL_key::TABLE,
t->s->db.str, t->s->table_name.str,
MDL_SHARED)));
/*
Prevent modifications to base tables if READ_ONLY is activated.
In any case, read only does not apply to temporary tables.
*/
if (!(flags & MYSQL_LOCK_IGNORE_GLOBAL_READ_ONLY) && !t->s->tmp_table)
{
if (t->reginfo.lock_type >= TL_WRITE_ALLOW_WRITE &&
enforce_ro && opt_readonly && !thd->slave_thread)
{
my_error(ER_OPTION_PREVENTS_STATEMENT, MYF(0),
opt_super_readonly ? "--read-only (super)" : "--read-only");
DBUG_RETURN(1);
}
}
}
/*
Locking of system tables is restricted:
locking a mix of system and non-system tables in the same lock
is prohibited, to prevent contention.
*/
if ((system_count > 0) && (system_count < count))
{
my_error(ER_WRONG_LOCK_OF_SYSTEM_TABLE, MYF(0));
DBUG_RETURN(1);
}
DBUG_RETURN(0);
}
5.7.14
点击(此处)折叠或打开
static int
lock_tables_check(THD *thd, TABLE **tables, size_t count, uint flags)
{
uint system_count= 0, i= 0;
/*
Identifies if the executed sql command can updated either a log
or rpl info table.
*/
bool log_table_write_query= false;
DBUG_ENTER("lock_tables_check");
log_table_write_query=
is_log_table_write_query(thd->lex->sql_command);
for (i=0 ; i
{
TABLE *t= tables[i];
/* Protect against 'fake' partially initialized TABLE_SHARE */
DBUG_ASSERT(t->s->table_category != TABLE_UNKNOWN_CATEGORY);
/*
Table I/O to performance schema tables is performed
only internally by the server implementation.
When a user is requesting a lock, the following
constraints are enforced:
*/
if (t->s->table_category == TABLE_CATEGORY_LOG &&
(flags & MYSQL_LOCK_LOG_TABLE) == 0 &&
!log_table_write_query)
{
/*
A user should not be able to prevent writes,
or hold any type of lock in a session,
since this would be a DOS attack.
*/
if (t->reginfo.lock_type >= TL_READ_NO_INSERT ||
thd->lex->sql_command == SQLCOM_LOCK_TABLES)
{
my_error(ER_CANT_LOCK_LOG_TABLE, MYF(0));
DBUG_RETURN(1);
}
}
if (t->reginfo.lock_type >= TL_WRITE_ALLOW_WRITE)
{
if (t->s->table_category == TABLE_CATEGORY_SYSTEM)
system_count++;
if (t->db_stat & HA_READ_ONLY)
{
my_error(ER_OPEN_AS_READONLY, MYF(0), t->alias);
DBUG_RETURN(1);
}
}
/*
If we are going to lock a non-temporary table we must own metadata
lock of appropriate type on it (I.e. for table to be locked for
write we must own metadata lock of MDL_SHARED_WRITE or stronger
type. For table to be locked for read we must own metadata lock
of MDL_SHARED_READ or stronger type).
*/
DBUG_ASSERT(t->s->tmp_table ||
thd->mdl_context.owns_equal_or_stronger_lock(MDL_key::TABLE,
t->s->db.str, t->s->table_name.str,
t->reginfo.lock_type >= TL_WRITE_ALLOW_WRITE ?
MDL_SHARED_WRITE : MDL_SHARED_READ));
/*
Prevent modifications to base tables if READ_ONLY is activated.
In any case, read only does not apply to temporary tables.
*/
if (!(flags & MYSQL_LOCK_IGNORE_GLOBAL_READ_ONLY) && !t->s->tmp_table)
{
if (t->reginfo.lock_type >= TL_WRITE_ALLOW_WRITE &&
check_readonly(thd, true))
DBUG_RETURN(1);
}
}
/*
Locking of system tables is restricted:
locking a mix of system and non-system tables in the same lock
is prohibited, to prevent contention.
*/
if ((system_count > 0) && (system_count < count))
{
my_error(ER_WRONG_LOCK_OF_SYSTEM_TABLE, MYF(0));
DBUG_RETURN(1);
}
DBUG_RETURN(0);
}
作者微信：
4万+
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
