一、自定义监控进程
写脚本,脚本要放到统一的位置
修改/usr/local/etc/zabbix_agentd.conf文件
UnsafeUserParameters=1
UserParameter=,
重启zabbix_agent
在web界面配置监控项和触发器
客户端
// 查看httpd进程,并监控他;如果没有进程,那就是没有启动进程,只要是启动了,最少都会有一个进程(进程数量>0,就代表这个进程没有问题,如果=0,那就说明有问题,要不是没有启动,要不就是没有这个进程)
[root@localhost ~]# ps -ef|grep httpd
root 1253153 1 0 14:27 ? 00:00:00 /usr/sbin/httpd -DFOREGROUND
apache 1253154 1253153 0 14:27 ? 00:00:00 /usr/sbin/httpd -DFOREGROUND
apache 1253155 1253153 0 14:27 ? 00:00:00 /usr/sbin/httpd -DFOREGROUND
apache 1253164 1253153 0 14:27 ? 00:00:00 /usr/sbin/httpd -DFOREGROUND
apache 1253185 1253153 0 14:27 ? 00:00:00 /usr/sbin/httpd -DFOREGROUND
root 1255081 3860 0 14:27 pts/1 00:00:00 grep --color=auto
//通过数量来判断,这个进程有没有问题,所以就要取他的数量,这个数量就是监控的指标
[root@localhost ~]# ps -ef|grep -v grep | grep httpd | wc -l // 过滤httpd,grep -v是取反,不显示grep --color=auto httpd这一行
5
//写一个脚本,判断这个进程是否大于零
[root@localhost ~]# mkdir /scripts //创建一个放脚本的目录
[root@localhost ~]# cd /scripts/
[root@localhost scripts]# ls
[root@localhost scripts]# chmod +x check_process.sh
[root@localhost scripts]# cat check_process.sh
#!/bin/bash
count=$(ps -ef | grep -Ev "grep|$0" | grep -c "$1")
if [ $count -eq 0 ];then
echo '1'
else
echo '0'
fi
[root@localhost scripts]# ./check_process.sh
0
// 去客户端的zabbix监控配置文件里面去加东西
[root@localhost ~]# cd /usr/local/etc/
[root@localhost etc]# ls
zabbix_agentd.conf zabbix_agentd.conf.d
[root@localhost etc]# vim zabbix_agentd.conf
322 UnsafeUserParameters=1 // 是否启用自定义监控项,可选值为{1|0},1是启用,0是关闭
326 # Format: UserParameter=<key>,<shell command> //把UserParameter=<key>,<shell command>复制到末尾
525 UserParameter=check_process[*],/scripts/check_process.sh $1
[root@localhost etc]# pkill zabbix
[root@localhost etc]# zabbix_agentd
[root@localhost etc]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 0.0.0.0:10050 0.0.0.0:*
服务端
[root@localhost ~]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 0.0.0.0:10050 0.0.0.0:*
LISTEN 0 128 0.0.0.0:10051 0.0.0.0:*
LISTEN 0 128 127.0.0.1:9000 0.0.0.0:*
LISTEN 0 128 0.0.0.0:111 0.0.0.0:*
LISTEN 0 128 0.0.0.0:80 0.0.0.0:*
LISTEN 0 32 192.168.122.1:53 0.0.0.0:*
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 5 127.0.0.1:631 0.0.0.0:*
LISTEN 0 80 *:3306 *:*
LISTEN 0 128 [::]:111 [::]:*
LISTEN 0 128 [::]:22 [::]:*
LISTEN 0 5 [::1]:631
[root@localhost ~]# zabbix_get -s 192.168.47.129 -k check_process[httpd]
0
加监控录像
1就代表有问题,就会产生报警,如下图(httpd服务停止的情况下)
httpd服务开启状态下,就是0
监控另外一个进程
二、自定义监控日志
写脚本,脚本要放到统一的位置
修改/usr/local/etc/zabbix_agentd.conf文件
UnsafeUserParameters=1
UserParameter=,
重启zabbix_agent
在web界面配置监控项和触发器
log.py
作用:检查日志文件中是否有指定的关键字
第一个参数为日志文件名(必须有,相对路径、绝对路径均可)
第二个参数为seek position文件的路径(可选项,若不设置则默认为/tmp/logseek文件。相对路径、绝对路径均可)
第三个参数为搜索关键字,默认为Error
[root@localhost scripts]# ls
check_process.sh log.py
[root@localhost scripts]# chmod +x log.py
[root@localhost scripts]# ls
check_process.sh log.py // 判断脚本
[root@localhost scripts]# ll /var/log/httpd/
总用量 4
-rw-r--r--. 1 root root 0 10月 8 14:27 access_log
-rw-r--r--. 1 root root 3474 10月 9 05:56 error_log
[root@localhost scripts]# yum -y install python3
[root@localhost scripts]# ./log.py /var/log/httpd/error_log //监控error_log
0
[root@localhost etc]# echo 'Error' >> /var/log/httpd/error_log
[root@localhost scripts]# ./log.py /var/log/httpd/error_log
1
[root@localhost etc]# rm -f /tmp/logseek
[root@localhost etc]# vim zabbix_agentd.conf
525 UserParameter=check_process[*],/scripts/check_process.sh $1
526 UserParameter=check_log[*],/scripts/log.py $1 $2 $3
[root@localhost etc]# pkill zabbix
[root@localhost etc]# zabbix_agentd
[root@localhost scripts]# chmod 755 /var/log/httpd/
服务端
[root@localhost ~]# zabbix_get -s 192.168.47.129 -k check_log[/var/log/httpd/error_log]
1
[root@zabbix ~]# zabbix_get -s 192.168.47.129 -k check_log[/var/log/httpd/error_log,/tmp/logseek,faild]
0
[root@zabbix ~]# zabbix_get -s 192.168.47.129 -k check_log[/var/log/httpd/error_log,/tmp/logseek,faild]
1
三、mysql主从
mysql安装
// 主库
[root@master ~]# yum -y install mariadb*
[root@master ~]# systemctl start mariadb //开机自启
[root@master ~]# systemctl disable --now firewall.service //关闭防火墙
[root@master ~]# getenforce 0 //关闭selinux
Disabled
[root@master ~]# ss -antl //确保3306端口已经监听起来
[root@master ~]# mysql //进入数据库
Welcome to the MariaDB monitor. Commands end with ; or \g.
Your MariaDB connection id is 8
Server version: 10.3.17-MariaDB MariaDB Server
Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
MariaDB [(none)]> set password = password("1"); //给数据库设置密码
Query OK, 0 rows affected (0.000 sec)
MariaDB [(none)]> quit
Bye
// 从库
[root@slave ~]# yum -y install mariadb*
[root@slave ~]# systemctl start mariadb //开机自启
[root@slave ~]# systemctl disable --now firewall.service //关闭防火墙[root@slave ~]# getenforce 0 //关闭selinux
Disabled
[root@slave ~]# ss -antl //确保3306端口已经监听起来
[root@slave ~]# mysql //进入数据库
Welcome to the MariaDB monitor. Commands end with ; or \g.
Your MariaDB connection id is 8
Server version: 10.3.17-MariaDB MariaDB Server
Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
MariaDB [(none)]> set password = password("1"); //给数据库设置密码
Query OK, 0 rows affected (0.000 sec)
MariaDB [(none)]> quit
Bye
确保从数据库与主数据库里的数据一样
//先查看主库有哪些库
[root@master ~]# mysql -uroot -p1 -e 'show databases;'
+--------------------+
| Database |
+--------------------+
| information_schema |
| mysql |
| performance_schema |
+--------------------+
//再查看从库有哪些库
[root@slave ~]# mysql -uroot -p1 -e 'show databases;'
+--------------------+
| Database |
+--------------------+
| information_schema |
| mysql |
| performance_schema |
+--------------------+
在主数据库里创建一个同步账号授权给从数据库使用
[root@master ~]# mysql -uroot -p1
Welcome to the MariaDB monitor. Commands end with ; or \g.
Your MariaDB connection id is 9
Server version: 10.3.17-MariaDB-log MariaDB Server
Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
MariaDB [(none)]> grant replication slave on *.* to 'repl'@'192.168.47.129' identified by 'repl123!';
Query OK, 0 rows affected (0.000 sec)
MariaDB [(none)]> flush privileges;
Query OK, 0 rows affected (0.000 sec)
MariaDB [(none)]> quit
Bye
配置主数据库
[root@master ~]# vim /etc/my.cnf.d/mariadb-server.cnf
16 [mysqld]
17 datadir=/var/lib/mysql
18 socket=/var/lib/mysql/mysql.sock
19 log-error=/var/log/mariadb/mariadb.log
20 pid-file=/run/mariadb/mariadb.pid
21
22 server-id = 10
23 log-bin = mysql_bin
[root@master ~]# ll /var/lib/mysql/
-rw-rw----. 1 mysql mysql 328 10月 10 16:08 mysql_bin.000001
-rw-rw----. 1 mysql mysql 19 10月 10 16:08 mysql_bin.index
MariaDB [(none)]> show master status;
+------------------+----------+--------------+------------------+
| File | Position | Binlog_Do_DB | Binlog_Ignore_DB |
+------------------+----------+--------------+------------------+
| mysql_bin.000001 | 328 | | |
+------------------+----------+--------------+------------------+
1 row in set (0.000 sec)
配置从数据库
[root@slave ~]# vim /etc/my.cnf.d/mariadb-server.cnf
16 [mysqld]
17 datadir=/var/lib/mysql
18 socket=/var/lib/mysql/mysql.sock
19 log-error=/var/log/mariadb/mariadb.log
20 pid-file=/run/mariadb/mariadb.pid
21
22 server-id = 20
23 relay-log = myrelay
[root@slave ~]# systemctl restart mariadb.service
//配置并启动主从复制
MariaDB [(none)]> change master to
-> master_host='192.168.47.128',
-> master_user='repl',
-> master_password='repl123!',
-> master_log_file='mysql_bin.000001',
-> master_log_pos=328;
Query OK, 0 rows affected (0.008 sec)
MariaDB [(none)]> start slave;
Query OK, 0 rows affected (0.001 sec)
MariaDB [(none)]> show slave status\G
*************************** 1. row ***************************
Slave_IO_State: Waiting for master to send event
Master_Host: 192.168.47.128
Master_User: repl
Master_Port: 3306
Connect_Retry: 60
Master_Log_File: mysql_bin.000001
Read_Master_Log_Pos: 328
Relay_Log_File: myrelay.000002
Relay_Log_Pos: 555
Relay_Master_Log_File: mysql_bin.000001
Slave_IO_Running: Yes
Slave_SQL_Running: Yes
Replicate_Do_DB:
Replicate_Ignore_DB:
Replicate_Do_Table:
Replicate_Ignore_Table:
Replicate_Wild_Do_Table:
Replicate_Wild_Ignore_Table:
Last_Errno: 0
Last_Error:
Skip_Counter: 0
Exec_Master_Log_Pos: 328
Relay_Log_Space: 856
Until_Condition: None
Until_Log_File:
Until_Log_Pos: 0
Master_SSL_Allowed: No
Master_SSL_CA_File:
Master_SSL_CA_Path:
Master_SSL_Cert:
Master_SSL_Cipher:
Master_SSL_Key:
Seconds_Behind_Master: 0
Master_SSL_Verify_Server_Cert: No
Last_IO_Errno: 0
Last_IO_Error:
Last_SQL_Errno: 0
Last_SQL_Error:
Replicate_Ignore_Server_Ids:
Master_Server_Id: 10
Master_SSL_Crl:
Master_SSL_Crlpath:
Using_Gtid: No
Gtid_IO_Pos:
Replicate_Do_Domain_Ids:
Replicate_Ignore_Domain_Ids:
Parallel_Mode: conservative
SQL_Delay: 0
SQL_Remaining_Delay: NULL
Slave_SQL_Running_State: Slave has read all relay log; waiting for the slave I/O thread to update it
Slave_DDL_Groups: 0
Slave_Non_Transactional_Groups: 0
Slave_Transactional_Groups: 0
1 row in set (0.000 sec)
在从服务器上编写脚本
[root@slave ~]# cd /scripts/
[root@slave scripts]# touch check_replication.sh
[root@slave scripts]# chmod +x check_replication.sh
[root@slave scripts]# ls
check_replication.sh
//为了让脚本更加安全,需要授权一个zabbix账户,除了要给select权限外还得给super,replication client权限
MariaDB [(none)]> grant select on *.* to 'zabbix'@'localhost' identified by 'zabbix123!';
Query OK, 0 rows affected (0.001 sec)
MariaDB [(none)]> flush privileges;
Query OK, 0 rows affected (0.000 sec)
MariaDB [(none)]> grant super,replication client on *.* to 'zabbix'@'localhost' identified by 'zabbix123!';
Query OK, 0 rows affected (0.000 sec)
MariaDB [(none)]> flush privileges;
Query OK, 0 rows affected (0.000 sec)
[root@slave scripts]# cat check_replication.sh
#!/bin/bash
count=$(mysql -uzabbix -p'zabbix123!' -e 'show slave status\G'|grep '_Running'|grep -c 'Yes')
if [ $count -ne 2 ];then
echo '1'
else
echo '0'
fi
// 如果在从服务器上关闭主从,再去执行脚本,就会显示1
MariaDB [(none)]> stop slave;
Query OK, 0 rows affected (0.002 sec)
[root@slave scripts]# ./check_replication.sh
1
[root@slave ~]# vim /usr/local/etc/zabbix_agentd.conf //修改zabbix_agentd.conf配置文件
322 UnsafeUserParameters=1
525 UserParameter=check_replication,/scripts/check_replication.sh //在配置文件末尾加上这一行
[root@slave ~]# pkill zabbix
[root@slave ~]# zabbix_agentd
[root@zabbix ~]# zabbix_get -s 192.168.47.129 -k check_replication //检查key是否可以用
0
把主从关闭的情况下
四、mysql主从延迟
MariaDB [(none)]> show slave status\G
*************************** 1. row ***************************
Slave_IO_State: Waiting for master to send event
Master_Host: 192.168.47.128
Master_User: repl
Master_Port: 3306
Connect_Retry: 60
Master_Log_File: mysql_bin.000001
Read_Master_Log_Pos: 328
Relay_Log_File: myrelay.000004
Relay_Log_Pos: 555
Relay_Master_Log_File: mysql_bin.000001
Slave_IO_Running: Yes
Slave_SQL_Running: Yes
Replicate_Do_DB:
Replicate_Ignore_DB:
Replicate_Do_Table:
Replicate_Ignore_Table:
Replicate_Wild_Do_Table:
Replicate_Wild_Ignore_Table:
Last_Errno: 0
Last_Error:
Skip_Counter: 0
Exec_Master_Log_Pos: 328
Relay_Log_Space: 1155
Until_Condition: None
Until_Log_File:
Until_Log_Pos: 0
Master_SSL_Allowed: No
Master_SSL_CA_File:
Master_SSL_CA_Path:
Master_SSL_Cert:
Master_SSL_Cipher:
Master_SSL_Key:
Seconds_Behind_Master: 0 //大于0,就代表有延迟;只要是=0或者是=NULL,就代表没有延迟
Master_SSL_Verify_Server_Cert: No
Last_IO_Errno: 0
Last_IO_Error:
Last_SQL_Errno: 0
Last_SQL_Error:
Replicate_Ignore_Server_Ids:
Master_Server_Id: 10
Master_SSL_Crl:
Master_SSL_Crlpath:
Using_Gtid: No
Gtid_IO_Pos:
Replicate_Do_Domain_Ids:
Replicate_Ignore_Domain_Ids:
Parallel_Mode: conservative
SQL_Delay: 0
SQL_Remaining_Delay: NULL
Slave_SQL_Running_State: Slave has read all relay log; waiting for the slave I/O thread to update it
Slave_DDL_Groups: 0
Slave_Non_Transactional_Groups: 0
Slave_Transactional_Groups: 0
1 row in set (0.000 sec)
// 写一个监控只从延迟的脚本
[root@slave scripts]# chmod +x check_replication_delay.sh
[root@slave scripts]# cat check_replication.sh
#!/bin/bash
delay=$(mysql -uzabbix -p'zabbix123!' -e 'show slave status\G'|grep 'Behind'|awk '{print $2}')
if [ $delay != NULL ];then //如果不是NULL就打印$delay,如果是就打印0
echo $delay
else
echo '0'
fi
[root@slave scripts]# ./check_replication_delay.sh
0
//修改zabbix_agentd.conf配置文件
[root@slave ~]# vim /usr/local/etc/zabbix_agentd.conf
525 UserParameter=check_replication,/scripts/check_replication.sh
526 UserParameter=check_replication_delay,/scripts/check_replicatio n_delay.sh
[root@slave ~]# pkill zabbix
[root@slave ~]# zabbix_agentd
[root@zabbix ~]# zabbix_get -s 192.168.47.129 -k check_replication_delay
0
创建监控项
五、声音报警
默认的zabbix只有七个不同的提示音,如果是想知道报警的事件,可以将不同类的触发器事件按不同的触发器级别来区分,比如可以把服务器宕机或网络故障定义为灾难级别,然后自己定义报警声音
六、用户和组权限的设置
[root@zabbix ~]# mysql -uroot -pwjj!
mysql: [Warning] Using a password on the command line interface can be insecure.
Welcome to the MySQL monitor. Commands end with ; or \g.
Your MySQL connection id is 3483
Server version: 5.7.34 MySQL Community Server (GPL)
Copyright (c) 2000, 2021, Oracle and/or its affiliates.
Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
mysql> show databases;
+--------------------+
| Database |
+--------------------+
| information_schema |
| mysql |
| performance_schema |
| sys |
| zabbix |
+--------------------+
5 rows in set (0.00 sec)
mysql> use zabbix;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A
Database changed
mysql> show tables; //在网页中看见的所有东西其实都在这个表里,这个表里面保存着所有的关于数据库和zabbix server里面的web界面的配置
+----------------------------+
| Tables_in_zabbix |
+----------------------------+
| acknowledges |
| actions |
| alerts |
| auditlog |
| auditlog_details |
| autoreg_host |
| conditions |
| config |
| config_autoreg_tls |
| corr_condition |
| corr_condition_group |
| corr_condition_tag |
| corr_condition_tagpair |
| corr_condition_tagvalue |
| corr_operation |
| correlation |
| dashboard |
| dashboard_page |
| dashboard_user |
| dashboard_usrgrp |
| dbversion |
| dchecks |
| dhosts |
| drules |
| dservices |
| escalations |
| event_recovery |
| event_suppress |
| event_tag |
| events |
| expressions |
| functions |
| globalmacro |
| globalvars |
| graph_discovery |
| graph_theme |
| graphs |
| graphs_items |
| group_discovery |
| group_prototype |
| history |
| history_log |
| history_str |
| history_text |
| history_uint |
| host_discovery |
| host_inventory |
| host_tag |
| hostmacro |
| hosts |
| hosts_groups |
| hosts_templates |
| housekeeper |
| hstgrp |
| httpstep |
| httpstep_field |
| httpstepitem |
| httptest |
| httptest_field |
| httptest_tag |
| httptestitem |
| icon_map |
| icon_mapping |
| ids |
| images |
| interface |
| interface_discovery |
| interface_snmp |
| item_condition |
| item_discovery |
| item_parameter |
| item_preproc |
| item_rtdata |
| item_tag |
| items |
| lld_macro_path |
| lld_override |
| lld_override_condition |
| lld_override_opdiscover |
| lld_override_operation |
| lld_override_ophistory |
| lld_override_opinventory |
| lld_override_opperiod |
| lld_override_opseverity |
| lld_override_opstatus |
| lld_override_optag |
| lld_override_optemplate |
| lld_override_optrends |
| maintenance_tag |
| maintenances |
| maintenances_groups |
| maintenances_hosts |
| maintenances_windows |
| media |
| media_type |
| media_type_message |
| media_type_param |
| module |
| opcommand |
| opcommand_grp |
| opcommand_hst |
| opconditions |
| operations |
| opgroup |
| opinventory |
| opmessage |
| opmessage_grp |
| opmessage_usr |
| optemplate |
| problem |
| problem_tag |
| profiles |
| proxy_autoreg_host |
| proxy_dhistory |
| proxy_history |
| regexps |
| report |
| report_param |
| report_user |
| report_usrgrp |
| rights |
| role |
| role_rule |
| script_param |
| scripts |
| service_alarms |
| services |
| services_links |
| services_times |
| sessions |
| sysmap_element_trigger |
| sysmap_element_url |
| sysmap_shape |
| sysmap_url |
| sysmap_user |
| sysmap_usrgrp |
| sysmaps |
| sysmaps_element_tag |
| sysmaps_elements |
| sysmaps_link_triggers |
| sysmaps_links |
| tag_filter |
| task |
| task_acknowledge |
| task_check_now |
| task_close_problem |
| task_data |
| task_remote_command |
| task_remote_command_result |
| task_result |
| timeperiods |
| token |
| trends |
| trends_uint |
| trigger_depends |
| trigger_discovery |
| trigger_queue |
| trigger_tag |
| triggers |
| users |
| users_groups |
| usrgrp |
| valuemap |
| valuemap_mapping |
| widget |
| widget_field |
+----------------------------+
166 rows in set (0.00 sec)
mysql> select * from users\G
*************************** 1. row ***************************
userid: 1
username: Admin
name: Zabbix
surname: Administrator
passwd: $2y$10$92nDno4n0Zm7Ej7Jfsz8WukBfgSS/U0QkIuu8WkJPihXBb2A1UrEK
url:
autologin: 1
autologout: 0
lang: zh_CN
refresh: 30s
theme: default
attempt_failed: 0
attempt_ip:
attempt_clock: 0
rows_per_page: 50
timezone: default
roleid: 3
*************************** 2. row ***************************
userid: 2
username: guest
name:
surname:
passwd: $2y$10$89otZrRNmde97rIyzclecuk6LwKAsHN0BcvoOKGjbT.BwMBfm7G06
url:
autologin: 0
autologout: 15m
lang: default
refresh: 30s
theme: default
attempt_failed: 0
attempt_ip:
attempt_clock: 0
rows_per_page: 50
timezone: default
roleid: 4
2 rows in set (0.00 sec)
mysql> select * from users\G
*************************** 1. row ***************************
userid: 1
username: Admin
name: Zabbix
surname: Administrator
passwd: $2y$10$92nDno4n0Zm7Ej7Jfsz8WukBfgSS/U0QkIuu8WkJPihXBb2A1UrEK
url:
autologin: 1
autologout: 0
lang: zh_CN
refresh: 30s
theme: default
attempt_failed: 0
attempt_ip:
attempt_clock: 0
rows_per_page: 50
timezone: default
roleid: 3
*************************** 2. row ***************************
userid: 2
username: guest
name:
surname:
passwd: $2y$10$89otZrRNmde97rIyzclecuk6LwKAsHN0BcvoOKGjbT.BwMBfm7G06
url:
autologin: 0
autologout: 15m
lang: default
refresh: 30s
theme: default
attempt_failed: 0
attempt_ip:
attempt_clock: 0
rows_per_page: 50
timezone: default
roleid: 4
*************************** 3. row ***************************
userid: 3
username: wangjingjing
name:
surname:
passwd: $2y$10$GGuUyAG8OtRjNR2ILkfKc.pl.ag4cLkquLM9ySFe6LrudYWQS60Dm
url:
autologin: 0
autologout: 0
lang: default
refresh: 30s
theme: default
attempt_failed: 0
attempt_ip:
attempt_clock: 0
rows_per_page: 50
timezone: default
roleid: 2
3 rows in set (0.00 sec)
mysql> select * from users_groups\G
*************************** 1. row ***************************
id: 4
usrgrpid: 7 //7是管理员
userid: 1
*************************** 2. row ***************************
id: 5
usrgrpid: 7
userid: 3 //3是刚刚创建的wangjingjing用户
*************************** 3. row ***************************
id: 2
usrgrpid: 8
userid: 2
*************************** 4. row ***************************
id: 3
usrgrpid: 9
userid: 2
4 rows in set (0.00 sec)