武汉网页设计_直播app软件开发_张家口seo_网站设计风格

BOT_NAME = 'python_city_data'

SPIDER_MODULES = ['python_city_data.spiders']
NEWSPIDER_MODULE = 'python_city_data.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'python_city_data (+http://www.yourdomain.com)'
#换伪造请求头
USER_AGENT = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
# Obey robots.txt rules
ROBOTSTXT_OBEY = False