# robots.txt for http://popsa.info
#+https://www.google.com/webmasters/tools/home?hl=ru
#+http://http://help.rambler.ru/article.html?s=221&id=327
#инструкций Allow в файлах robots.txt не бывает.
#Использование регулярных выражений не предусмотрено Disallow: *"не запрещают ничего, поскольку файлов со звездочкой не существует.
#Если директива Disallow будет пустой, это значит, что робот может индексировать ВСЕ файлы

#User-agent: Googlebot
#User-agent: StackRambler
User-agent: *
Disallow: /index
Disallow: /krugozor/
Disallow: /forum/profile.cgi
Disallow: /forum/printpage.cgi
Disallow: /forum/search2.cgi
Disallow: /forum/messenger.cgi
Disallow: /forum/register.cgi
Disallow: /forum/loginout.cgi
Disallow: /forum/whosonline.cgi
Disallow: /forum/chat.cgi
Disallow: /forum/help.cgi
Disallow: /forum/privacy.cgi
Disallow: /forum/post.cgi
Disallow: /forum/postings.cgi
Disallow: /forum/misc.cgi
Disallow: /forum/subscr.cgi
Disallow: /cgi-bin/
Disallow: /stat/
Host: popsa.info
Sitemap: http://popsa.info/sitemap.xml


#+http://help.live.com/help.aspx?mkt=ru-RU&project=wl_webmasters
User-agent: msnbot 
Disallow: /index.shtml
Disallow: /krugozor/
Disallow: /forum/profile.cgi
Disallow: /forum/printpage.cgi
Disallow: /forum/search2.cgi
Disallow: /forum/messenger.cgi
Disallow: /forum/register.cgi
Disallow: /forum/loginout.cgi
Disallow: /forum/whosonline.cgi
Disallow: /forum/chat.cgi
Disallow: /forum/help.cgi
Disallow: /forum/privacy.cgi
Disallow: /forum/post.cgi
Disallow: /forum/postings.cgi
Disallow: /forum/misc.cgi
Disallow: /forum/subscr.cgi
Disallow: /cgi-bin/
Disallow: /stat/
Disallow: /*.gif$
Disallow: /*.mp3$
Disallow: /*.avi$
Disallow: /*.mpg$
Crawl-delay: 60
Sitemap: http://popsa.info/sitemap.xml

#+http://help.yahoo.com/l/us/yahoo/search/webcrawler/
#https://siteexplorer.search.yahoo.com/
User-agent: Slurp
Allow: /$
Allow: /content.html
Allow: /links.html
Allow: /technology.html
Allow: /bio/
Allow: /demo/
Allow: /forum/$
Allow: /forum/forums.cgi
Disallow: /
Host: popsa.info
Sitemap: http://popsa.info/sitemap.xml

# http://www.alexa.com/help/webmasters
# http://web.archive.org/collections/web/faqs.html#exclusions
# User-agent: ia_archiver


# http://www.cuil.com/info/webmaster_info/
User-agent: twiceler
Allow: /$
Allow: /content.html
Allow: /links.html
Allow: /technology.html
Allow: /bio/
Allow: /demo/
Disallow: /
Crawl-delay: 900
Host: popsa.info

# http://www.dotnetdotcom.org/
User-agent: dotbot 
Allow: /$
Allow: /content.html
Allow: /links.html
Allow: /technology.html
Allow: /bio/
Allow: /demo/
Disallow: /
Crawl-delay: 900
Host: popsa.info

#User-agent: myhtdig
#User-agent: msnbot-media
#User-agent: libwww
#User-agent: wget
Disallow: /

#+http://webmaster.yandex.ru/
User-agent: Yandex
Allow: /$
Allow: /bio/
Allow: /demo/
Allow: /smi/
Allow: /tex/
Allow: /forum/
Allow: /content.html$
Allow: /links.html$
Allow: /technology.html$
Allow: /forum/forums.cgi
Allow: /forum/topic.cgi
Disallow: /
Clean-param: show /forum/forums.cgi
Clean-param: start /forum/topic.cgi
Host: popsa.info
Sitemap: http://popsa.info/sitemap.xml