1:为了让搜索引擎更好的抓取内容,或者拒绝搜索引擎爬内容。
把 robot.txt 放在根目录就好了。
urls.py
from django.conf.urls.defaults import *
from django.conf import settings
from django.http import HttpResponse
urlpatterns = patterns("",
(r"^static/(?P<path>.*)$", "django.views.static.serve", {"document_root": settings.STATIC_DOC_ROOT}),
(r"^$", "views.index"),
(r"^robots\.txt$", lambda r: HttpResponse("User-agent: *\nDisallow: /", mimetype="text/plain")),
(r"^CCSN\.txt$", lambda r: HttpResponse("CCWSN00181", mimetype="text/plain")),
)