blob: d7306cdcf812c8faafb5eb4f865b4cf8f7c88c22 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
|
# robots.txt file for webtrees
#
# Copyright (C) Greg Roach 2014
#
# This file needs to be placed in the domain root folder,
# such as "www.example.com/robots.txt". It will not work in a
# subfolder, such as "www.example.com/webtrees/robots.txt"
# If you need to move it, then remember to adjust the paths as well.
# e.g. "Allow: /index.php" becomes "Allow: /webtrees/index.php".
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
# Well-behaved robots understand "Allow" directives.
# Google (plus some others?) understands the "$" pattern match
User-Agent: bingbot # Bing
User-Agent: Googlebot # Google
User-Agent: ia_archiver # Internet Archive "wayback" machine
User-Agent: msnbot # Microsoft
User-Agent: robozilla # DMOZ
User-Agent: slurp # Yahoo!
User-Agent: teoma # Ask Jeeves
User-Agent: yandex # Popular Russian search engine
User-Agent: baiduspider # Popular Chinese search engine
Allow: /family.php
Allow: /index.php
Allow: /indilist.php
Allow: /individual.php
Allow: /mediafirewall.php
Allow: /mediaviewer.php
Allow: /module.php?mod=sitemap
Allow: /note.php
Allow: /repo.php
Allow: /site-offline.php
Allow: /site-unavailable.php
Allow: /source.php
Allow: /$
Disallow: /
# Disallow all other robots
User-Agent: *
Disallow: /
# Put your sitemap here:
#Sitemap: http://www.example.com/module.php?mod=sitemap&mod_action=generate&file=sitemap.xml
|
