# robots.txt for FIDH Archive Site
# This is a public archive - all content should be crawlable

User-agent: *
Allow: /

# Allow access to AI instructions
Allow: /AI.txt

# Block sensitive directories and files
Disallow: /.git/
Disallow: /.github/
Disallow: /.htaccess
Disallow: /.htpasswd/
Disallow: /.env
Disallow: /*.env*
Disallow: /*.log$
Disallow: /*.db$
Disallow: /*.sqlite$
Disallow: /node_modules/
Disallow: /vendor/
Disallow: /bower_components/

# Sitemap location
Sitemap: http://wearenotafraid.fidh.org/sitemap.xml

# Crawl delay for politeness
Crawl-delay: 5