blob: 60637032696d5cb6e1150189b6e5f7d3a02a396a (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
|
# webtrees: online genealogy
# Copyright (C) 2015 webtrees development team
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This file needs to be placed in the domain root folder,
# such as "www.example.com/robots.txt". It will not work in a
# subfolder, such as "www.example.com/webtrees/robots.txt"
# If you need to move it, then remember to adjust the paths as well.
# e.g. "Allow: /index.php" becomes "Allow: /webtrees/index.php".
#
# Well-behaved robots understand "Allow" directives.
# Google (plus some others?) understands the "$" pattern match
User-Agent: bingbot # Bing
User-Agent: Googlebot # Google
User-Agent: ia_archiver # Internet Archive "wayback" machine
User-Agent: msnbot # Microsoft
User-Agent: robozilla # DMOZ
User-Agent: slurp # Yahoo!
User-Agent: teoma # Ask Jeeves
User-Agent: yandex # Popular Russian search engine
User-Agent: baiduspider # Popular Chinese search engine
Allow: /family.php
Allow: /index.php
Allow: /indilist.php
Allow: /individual.php
Allow: /mediafirewall.php
Allow: /mediaviewer.php
Allow: /module.php?mod=sitemap
Allow: /note.php
Allow: /repo.php
Allow: /site-offline.php
Allow: /site-unavailable.php
Allow: /source.php
Allow: /$
Disallow: /
# Disallow all other robots
User-Agent: *
Disallow: /
# Put your sitemap here:
#Sitemap: http://www.example.com/module.php?mod=sitemap&mod_action=generate&file=sitemap.xml
|
