User-agent: Scrapy
Allow: /

# Allow facebookexternalhit
User-agent: facebookexternalhit
Disallow:

# Block all other robots
User-agent: *
Disallow: /