centos7.2分布式部署pyspider

1.环境

系统：Linux 3.10.0-862.2.3.el7.x86_64

python版本：Python 3.5.2 :: Anaconda 4.2.0 (64-bit)

2.搭建

2.1搭建python3

采坑后，选择集成环境Anaconda（其在Linux平台下（通过conda管理）安装更方便）

可查阅：winpython, anaconda 哪个更好？

2.1.1编译

# 下载依赖
yum install -y ncurses-devel openssl openssl-devel zlib-devel gcc make glibc-devel libffi-devel glibc-static glibc-utils sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-deve
# 下载python版本
wget https://www.python.org/ftp/python/3.5.2/Python-3.5.2.tgz
# 或者使用国内源
wget http://mirrors.sohu.com/python/3.5.2/Python-3.5.2.tgz
mv Python-3.5.2.tgz /usr/local/src
cd /usr/local/src
# 解压
tar -zxf Python-3.5.2.tgz
cd Python-3.5.2
# 编译安装
./configure --prefix=/usr/local/python3.5 --enable-shared
make && make install
# 建立软链接
ln -s /usr/local/python3.5/bin/python3 /usr/bin/python3
echo "/usr/local/python3.5/lib" > /etc/ld.so.conf.d/python3.5.conf
ldconfig
# 验证python3
python3
Python 3.5.2 |Anaconda 4.2.0 (64-bit)| (default, Jul  2 2016, 17:53:06) 
[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> 

2.1.2集成环境anaconda

# 集成环境anaconda(推荐)
wget https://repo.continuum.io/archive/Anaconda3-4.2.0-Linux-x86_64.sh
# 直接安装即可
./Anaconda3-4.2.0-Linux-x86_64.sh
# 若提示权限不够，添加可执行权限
chmod 777 Anaconda3-4.2.0-Linux-x86_64.sh
# 若出错，可能是解压失败
yum install bzip2

#使环境变量配置生效  
source ~/.bashrc

2.1.3安装pyspider

# 搭建虚拟环境sbrid python版本3.*
conda create -n sbrid python=3*
# 进入环境
source activate sbrid
#安装 pycurl（否则报错）
conda install pycurl
# 安装pyspider
pip install pyspider
# 安装mysql访问组件
pip install mysql-connector-python
#安装redis访问组件
pip install redis
#升级pip
pip install --upgrade pip
#关闭环境
source deactivate sbrid

2.1.4创建mysql用户

# 登录
mysql -u root -p
# 创建一个用户 自己设定账户密码
CREATE USER 'user_name'@'localhost' IDENTIFIED BY 'user_pass';
GRANT ALL PRIVILEGES ON *.* TO 'use