linux下安装并运行scrapyd

linux下安装并运行scrapyd(python3及以上)

安装

pip3 isntall scrapyd

配置文件

新建存放配置文件文件夹

mkdir /etc/scrapyd

编辑配置文件

vim /etc/scrapyd/scrapyd.conf
写入（[官网示例](https://scrapyd.readthedocs.io/en/stable/config.html)）：
[scrapyd]
eggs_dir    = eggs
logs_dir    = logs
items_dir   =
jobs_to_keep = 5
dbs_dir     = dbs
max_proc    = 0
max_proc_per_cpu = 10
finished_to_keep = 100
poll_interval = 5.0
bind_address = 0.0.0.0
http_port   = 6800
debug       = off
runner      = scrapyd.runner
application = scrapyd.app.application
launcher    = scrapyd.launcher.Launcher
webroot     = scrapyd.website.Root

[services]
schedule.json     = scrapyd.webservice.Schedule
cancel.json       = scrapyd.webservice.Cancel
addversion.json   = scrapyd.webservice.AddVersion
listprojects.json = scrapyd.webservice.ListProjects
listversions.json = scrapyd.webservice.ListVersions
listspiders.json  = scrapyd.webservice.ListSpiders
delproject.json   = scrapyd.webservice.DeleteProject
delversion.json   = scrapyd.webservice.DeleteVersion
listjobs.json     = scrapyd.webservice.ListJobs
daemonstatus.json = scrapyd.webservice.DaemonStatus

开放端口6800

可参考(https://blog.csdn.net/qq_38351453/article/details/116024783)

启动

设置软连接
ln -s /usr/local/python3/bin/scrapyd /usr/bin/scrapyd
启动
（scrapyd > /dev/null &）
也可在python scrapyd的安装包目录下执行 scrapyd

访问服务器6800端口即可 如 127.0.0.1:6800

设置为系统后台服务

新建目录 mkdir /var/scrapyd
新建文件/etc/init.d/scrapyd

touch /etc/init.d/scrapyd

设置权限

sudo chmod 755 /etc/init.d/scrapyd

编辑文件

vi /etc/init.d/scrapyd
#!/bin/bash
# chkconfig: 2345 20 80
# description: Srapyd
PORT=6800
HOME="/var/scrapyd"
BIN="/usr/bin/scrapyd"
 
pid=`netstat -lnopt | grep :$PORT | awk '/python/{gsub(/\/python/,"",$7);print $7;}'`
start() {
   if [ -n "$pid" ]; then
      echo "server already start,pid:$pid"
      return 0
   fi
 
   cd $HOME
   nohup $BIN >> $HOME/scrapyd.log 2>&1 &
   echo "start at port:$PORT"
}
 
stop() {
   if [ -z "$pid" ]; then
      echo "not find program on port:$PORT"
      return 0
   fi
 
   #结束程序，使用讯号2，如果不行可以尝试讯号9强制结束
   kill -9 $pid
   echo "kill program use signal 9,pid:$pid"
}
 
status() {
   if [ -z "$pid" ]; then
      echo "not find program on port:$PORT"
   else
      echo "program is running,pid:$pid"
   fi
}
 
case $1 in
   start)
      start
   ;;
   stop)
      stop
   ;;
   status)
      status
   ;;
   *)
      echo "Usage: {start|stop|status}"
   ;;
esac
 
exit 0

命令操作（启动停止状态）

service scrapyd {start|stop|status}

设置为系统启动项

chkconfig --add scrapyd
chkconfig --del scrapyd
chkconfig --list

**若远程访问则需要开启服务器6800端口安全组