#用virtualenv隔离出一个用于scrapy的python运行环境,如何安装见http://www.xsdou.com/1769.html
virtualenv -p /home/hadoop-user/python27/bin/python virtual_scrapy_python
cd virtual_scrapy_python/
source bin/activate
pip install scrapy
#根据scrapy官网首页的Sample Scrapy Code建立一个spider样例代码并运行,运行会有几个包的缺少,按照下面的方法进行安装即可
#缺少twisted
pip install twisted
pip install w3lib
#缺少lxml
pip install lxml
安装lxml可能会发生错误
根据这篇文章执行
yum install python-devel libxml2-devel libxslt-devel
yum install pyOpenSSL
即可
#缺少cssselect
pip install cssselect
#缺少cryptography
pip install cryptography
安装cryptography可能发生错误
根据这篇文章执行
yum install gcc libffi-devel python-devel openssl-devel
即可