# robots.txt - Generic fs-websites 21/02/2010

# Match all bots
User-agent: *
# DISALLOW - All folders.
# Disallow: /
Allow: /


# DISALLOW - Standard folder set.
Disallow: /files/
Disallow: /forum/
Disallow: /images/
Disallow: /maintenance/
Disallow: /mkportal/

# Following is commented to ensure search engines get the redirect.
# Disallow: /resources/
Disallow: /resources/images/
Disallow: /resources/offline/



# DISALLOW - Non-standard folder set.

# DISALLOW - FREEflow folder set.
Disallow: /forum2/
Disallow: /Admin/
Disallow: /beta/
Disallow: /bugzilla/
Disallow: /cgi-bin/
Disallow: /cgiirc/
Disallow: /downloads/
Disallow: /FFBeta/
Disallow: /ffwiki/
Disallow: /flyspray/
Disallow: /forum_phpbb/
Disallow: /intercom/
Disallow: /Misc_HTML/
Disallow: /pages/
Disallow: /phpgrid/
Disallow: /pjirc/
Disallow: /screenshots/
Disallow: /videos/

# CRAWL - Time/frequency related
Crawl-Delay: 600
Request-rate: 1/300     # maximum rate is 1 page every 300 seconds
Visit-time: 0600-0845   # only visit between 06:00 and 08:45 UTC (GMT)