robots.txt de Le bon coin

De Vulgum Techus
Version du 21 septembre 2016 à 15:45 par Admin2 (discuter | contributions)

(diff) ← Version précédente | Voir la version courante (diff) | Version suivante → (diff)
Aller à : Navigation, rechercher

robot.txt au 21 septembre 2016

## It's forbidden to use search robots or other automatic methods to access Leboncoin.fr.
## Access is only permitted with special permission from Leboncoin.fr.

## Google
User-agent: Googlebot
# friendly exceptions
Allow: /aide.htm
Allow: /aide_pro.htm
Allow: /arts_de_la_table
Allow: /materiel
Allow: /services
Allow: /sports_hobbies
# www2 only
Disallow: /ai
Disallow: /ar
Disallow: /cp
Disallow: /ma
Disallow: /rd
Disallow: /se
Disallow: /sp
Disallow: /support
Disallow: /dc/decisions_de_justice
# all
Disallow: /*th=0
Disallow: /aw
Disallow: /st
Disallow: /maintenance.html
Disallow: /pub
Disallow: /boutique/*/*print=0
Disallow: /boutique/*.htm?*ca=*_k&*
Disallow: /popup_siege_social.htm
Disallow: /dc/popup_siege_social
Disallow: /*_mediametrie
Allow: /* 

User-agent: Mediapartners-Google
# friendly exceptions
Allow: /aide.htm
Allow: /aide_pro.htm
Allow: /arts_de_la_table
Allow: /materiel
Allow: /services
Allow: /sports_hobbies
# www2 only
Disallow: /ai
Disallow: /ar
Disallow: /cp
Disallow: /ma
Disallow: /rd
Disallow: /se
Disallow: /sp
Disallow: /support
Disallow: /dc/decisions_de_justice
# all
Disallow: /*th=0
Disallow: /aw
Disallow: /st
Disallow: /maintenance.html
Disallow: /pub
Disallow: /boutique/*/*print=0
Disallow: /popup_siege_social.htm
Disallow: /dc/popup_siege_social
Disallow: /*_mediametrie
Allow: /*

## Yahoo
User-agent: slurp
# friendly exceptions
Allow: /aide.htm
Allow: /aide_pro.htm
Allow: /arts_de_la_table
Allow: /materiel
Allow: /services
Allow: /sports_hobbies
# www2 only
Disallow: /ai
Disallow: /ar
Disallow: /cp
Disallow: /ma
Disallow: /rd
Disallow: /se
Disallow: /sp
Disallow: /support
Disallow: /dc/decisions_de_justice
# all
Disallow: /*th=0
Disallow: /aw
Disallow: /st
Disallow: /maintenance.html
Disallow: /pub
Disallow: /boutique/*/*print=0
Disallow: /popup_siege_social.htm
Disallow: /dc/popup_siege_social
Disallow: /*_mediametrie
Allow: /*

# Googlebot Mobile
User-agent: Googlebot-Mobile
Disallow: /popup_siege_social.htm
Disallow: /dc/popup_siege_social
Allow: /*

User-agent: msnbot
# friendly exceptions
Allow: /aide.htm
Allow: /aide_pro.htm
Allow: /arts_de_la_table
Allow: /materiel
Allow: /services
Allow: /sports_hobbies
# www2 only
Disallow: /ai
Disallow: /ar
Disallow: /cp
Disallow: /ma
Disallow: /rd
Disallow: /se
Disallow: /sp
Disallow: /support
Disallow: /dc/decisions_de_justice
# all
Disallow: /*th=0
Disallow: /aw
Disallow: /st
Disallow: /maintenance.html
Disallow: /pub
Disallow: /boutique/*/*print=0
Disallow: /popup_siege_social.htm
Disallow: /*_mediametrie
Allow: /*

# allow archive.org to screenshot all our pages
User-agent: ia_archiver
Disallow:

User-agent: Speobot
Disallow: /

User-agent: *
Disallow: /