# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

Host: https://gildor.org

User-agent: *

# Directories
# comment "disallow" for some theme-related things, otherwise google thinks that that site is not mobile-friendly
Disallow: /scripts/
Disallow: /updates/
Disallow: /profiles/
# Files
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /search/
# Custom paths
Disallow: /down/
Disallow: /donate
Disallow: /en/donate

# Forum
#Disallow: /smf/index.php?action=printpage* - invisible for bots
#Disallow: /smf/index.php?action=profile* - ...
#Disallow: /smf/index.php?action=login - ...
#Disallow: /smf/index.php?action=url
#Disallow: /smf/index.php?action=help
Disallow: /smf/index.php?action=search
Disallow: /smf/index.php?action=stats
#Disallow: /smf/index.php?action=recent - disabled with meta noindex
#Disallow: /smf/index.php?topic*new - now has canonical URL, so doesn't matter
Disallow: /smf/*type=rss*
# Mobile version of forum, terrible one
#Disallow: /smf/*wap* -- redirecting to no-wap2 page with 301 code
Disallow: /smf/*imode*

# Heavy site loading
User-agent: serpstatbot
Disallow: /
# this bot has nearly the same IP as serpstatbot, seems same group, and doing same heavy load of the site
User-Agent: trendictionbot
Disallow: /
# made 6.5k requests during night
User-agent: ZumBot
Disallow: /

User-agent: bingbot
Crawl-delay: 10

User-agent: Yandex
Crawl-delay: 10

User-agent: Qwantify
Crawl-delay: 20

# Youtube link scanner - generates 404 errors when someone uploads a video with site refs
# https://sudofox.hatenablog.com/entry/google-is-scanning-for-and-crawling-urls-in-your-private-youtube-videos
User-agent: Google-Youtube-Links
Disallow: /