robots.txt 741 B

12345678910111213141516171819202122232425262728293031323334
  1. #
  2. # robots.txt
  3. #
  4. # This file is to prevent the crawling and indexing of certain parts
  5. # of your site by web crawlers and spiders run by sites like Yahoo!
  6. # and Google. By telling these "robots" where not to go on your site,
  7. # you save bandwidth and server resources.
  8. #
  9. #
  10. # For more information about the robots.txt standard, see:
  11. # http://www.robotstxt.org/wc/robots.html
  12. #
  13. # For syntax checking, see:
  14. # http://www.sxw.org.uk/computing/robots/check.html
  15. User-Agent: *
  16. # Directories
  17. Disallow: /app/
  18. Disallow: /assets/
  19. Disallow: /bin/
  20. Disallow: /config/
  21. Disallow: /documentation/
  22. Disallow: /home/
  23. Disallow: /main/
  24. Disallow: /plugin/
  25. Disallow: /src/
  26. Disallow: /tests/
  27. Disallow: /vendor/
  28. # Files
  29. Disallow: /license.txt
  30. Disallow: /README.txt