Priezastys:
- klientai
- linkai
- hakeriai
Naudojam robot.txt
# eina komantarai
User-agent: * # reikšia visies taikoma
Disallow: /doc #Don’t spider anything in the doc dir
Disallow:/secweb/ # Don’t spider anything in indef files (default.htm, index.html, index.php)
Disallow: /config.php #neliesti šio failo
