# All robots will spider the domain

User-Agent: *
Disallow: /offers
Allow: /


User-Agent: googlebot
Disallow: /offers
Allow: /

User-Agent: *
Disallow: /offers2
Allow: /


User-Agent: googlebot
Disallow: /offers2
Allow: /

User-Agent: *
Disallow: /offers3
Allow: /


User-Agent: googlebot
Disallow: /offers3
Allow: /


User-Agent: *
Disallow: /offers4
Allow: /


User-Agent: googlebot
Disallow: /offers4
Allow: /

# Disallow directory /cgi-bin/

User-agent: *
Disallow: /cgi-bin/

User-agent: *
Disallow: /frame/

User-agent: *
Disallow: /offers

User-agent: *
Disallow: /pe/