diff options
Diffstat (limited to 'robots-example.txt')
| -rw-r--r-- | robots-example.txt | 2 |
1 files changed, 2 insertions, 0 deletions
diff --git a/robots-example.txt b/robots-example.txt index eb8ea7bee0..d7306cdcf8 100644 --- a/robots-example.txt +++ b/robots-example.txt @@ -23,6 +23,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA # Well-behaved robots understand "Allow" directives. +# Google (plus some others?) understands the "$" pattern match User-Agent: bingbot # Bing User-Agent: Googlebot # Google User-Agent: ia_archiver # Internet Archive "wayback" machine @@ -44,6 +45,7 @@ Allow: /repo.php Allow: /site-offline.php Allow: /site-unavailable.php Allow: /source.php +Allow: /$ Disallow: / # Disallow all other robots |