# www.robotstxt.org/ # www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449 User-agent: * Disallow: /treecode/ Disallow: /*/treecode/ Disallow: /it/treecode/ Disallow: /en/treecode/ Disallow: /de/treecode/ Disallow: /fr/treecode/ Disallow: /es/treecode/ Disallow: /it/business/e-card/ Disallow: /en/business/e-card/ Disallow: /de/business/e-card/ Disallow: /fr/business/e-card/ Disallow: /es/business/e-card/ # --- blocking aggressive bots # Huawei search service User-agent: PetalBot Disallow: / # Russian search engine User-agent: Yandex Disallow: / # Unused SEO tool User-agent: AhrefsBot Disallow: / # Unused SEO tool User-agent: SemrushBot Disallow: / # Unused SEO tool User-agent: DataForSeoBot Disallow: / # Image crawler/stealer User-agent: ImagesiftBot Disallow: / # Bing bot is needed but too aggressive, slowing it down User-agent: bingbot Crawl-delay: 1 Sitemap: https://www.treedom.net/sitemap.xml