# robots.txt - robot exclusion file - back-end server version - no robots! # ======================================================================== User-agent: htdig/3.1.6 Disallow: /htbin-post Disallow: /cgi-bin/BLAST Disallow: /entrez Disallow: /cgi-bin/COG Disallow: /Entrez Disallow: /genmap Disallow: /LocusLink Disallow: /cgi-bin/Entrez Disallow: /sutils Disallow: /Taxonomy/Browser Disallow: /Taxonomy/Selector Disallow: /Taxonomy/CommonTree Disallow: /entrez/sutils Disallow: /geo/query Disallow: /mapview Disallow: /blast/d Disallow: /blast/D Disallow: /blast/g Disallow: /blast/BlastAlign.cgi Disallow: /blast/bl2seq/wblast2.cgi Disallow: /sites Disallow: /portal Disallow: /pmc/utilities/ Disallow: /pmc/issues/ Disallow: /pmc/articles/ Disallow: /pmc/ivip/ Disallow: /pmc/articlerender.fcgi Disallow: /pmc/pagerender.fcgi Disallow: /pmc/picrender.fcgi Disallow: /pmc/tocrender.fcgi Disallow: /pmc/*report= Disallow: /pmc/*page= Disallow: /pmc/articles/*/citedby/ User-agent: Googlebot Disallow: /htbin-post Disallow: /cgi-bin/BLAST Disallow: /cgi-bin/COG Disallow: /COG Disallow: /genmap Disallow: /LocusLink Disallow: /mailman Disallow: /cgi-bin/Entrez Disallow: /sutils Disallow: /entrez/viewer.fcgi Disallow: /Taxonomy/Browser Disallow: /Taxonomy/Selector Disallow: /Taxonomy/CommonTree Disallow: /entrez/sutils Disallow: /geo/query Disallow: /genome Disallow: /Genomes Disallow: /GENOMES Disallow: /SAGE Disallow: /SNP Disallow: /IEB/ToolBox/SB Disallow: /Structure/dgs Disallow: /Structure/Entrez Disallow: /Structure/VA Disallow: /Structure/cblast/cblast.cgi Disallow: /Coffeebreak Disallow: /gorf Disallow: /genomes Disallow: /mapview Disallow: /blast/d Disallow: /blast/D Disallow: /blast/g Disallow: /blast/BlastAlign.cgi Disallow: /blast/bl2seq/wblast2.cgi Disallow: /entrez/eutils Disallow: /entrez/utils Disallow: /portal Disallow: /pmc/utilities/ Disallow: /pmc/*/bin/ Disallow: /pmc/*/pdf/ Disallow: /pmc/*/equ/ Disallow: /pmc/*.cgi Disallow: /pmc/*report= Disallow: /pmc/*page= Disallow: /pmc/articles/*/citedby/ User-agent: Teleport Pro Disallow: /htbin-post Disallow: /cgi-bin/BLAST Disallow: /entrez Disallow: /cgi-bin/COG Disallow: /Entrez Disallow: /genmap Disallow: /LocusLink Disallow: /mailman Disallow: /genome Disallow: /cgi-bin/Entrez Disallow: /sutils Disallow: /Taxonomy/Browser Disallow: /Taxonomy/Selector Disallow: /Taxonomy/CommonTree Disallow: /entrez/sutils Disallow: /geo/query Disallow: /mapview Disallow: /blast/d Disallow: /blast/D Disallow: /blast/g Disallow: /blast/BlastAlign.cgi Disallow: /blast/bl2seq/wblast2.cgi Disallow: /sites Disallow: /portal Disallow: /pmc/utilities/ Disallow: /pmc/issues/ Disallow: /pmc/articles/ Disallow: /pmc/ivip/ Disallow: /pmc/articlerender.fcgi Disallow: /pmc/pagerender.fcgi Disallow: /pmc/picrender.fcgi Disallow: /pmc/tocrender.fcgi User-agent: yacybot Disallow: * User-agent: Browsershots Disallow: User-agent: Slurp Crawl-delay: 1 Disallow: /htbin-post Disallow: /cgi-bin/BLAST Disallow: /entrez Disallow: /cgi-bin/COG Disallow: /Entrez Disallow: /genmap Disallow: /LocusLink Disallow: /mailman Disallow: /cgi-bin/Entrez Disallow: /sutils Disallow: /Taxonomy/Browser Disallow: /Taxonomy/Selector Disallow: /Taxonomy/CommonTree Disallow: /entrez/sutils Disallow: /geo/query Disallow: /mapview Disallow: /blast/d Disallow: /blast/D Disallow: /blast/g Disallow: /blast/BlastAlign.cgi Disallow: /blast/bl2seq/wblast2.cgi Disallow: /sites Disallow: /portal Disallow: /pmc/utilities/ Disallow: /pmc/*/bin/ Disallow: /pmc/*/pdf/ Disallow: /pmc/*/equ/ Disallow: /pmc/*.cgi Disallow: /pmc/*report= Disallow: /pmc/*page= Disallow: /pmc/articles/*/citedby/ User-agent: MSNBot Disallow: /htbin-post Disallow: /cgi-bin/BLAST Disallow: /entrez Disallow: /cgi-bin/COG Disallow: /Entrez Disallow: /genmap Disallow: /LocusLink Disallow: /mailman Disallow: /cgi-bin/Entrez Disallow: /sutils Disallow: /Taxonomy/Browser Disallow: /Taxonomy/Selector Disallow: /Taxonomy/CommonTree Disallow: /entrez/sutils Disallow: /geo/query Disallow: /mapview Disallow: /blast/d Disallow: /blast/D Disallow: /blast/g Disallow: /blast/BlastAlign.cgi Disallow: /blast/bl2seq/wblast2.cgi Disallow: /sites Disallow: /portal Disallow: /pmc/utilities/ Disallow: /pmc/*/bin/ Disallow: /pmc/*/pdf/ Disallow: /pmc/*/equ/ Disallow: /pmc/*.cgi Disallow: /pmc/*report= Disallow: /pmc/*page= Disallow: /pmc/articles/*/citedby/ User-agent: * Disallow: /htbin-post Disallow: /cgi-bin/BLAST Disallow: /entrez Disallow: /cgi-bin/COG Disallow: /Entrez Disallow: /genmap Disallow: /LocusLink Disallow: /mailman Disallow: /cgi-bin/Entrez Disallow: /sutils Disallow: /Taxonomy/Browser Disallow: /Taxonomy/Selector Disallow: /Taxonomy/CommonTree Disallow: /entrez/sutils Disallow: /geo/query Disallow: /mapview Disallow: /blast/d Disallow: /blast/D Disallow: /blast/g Disallow: /blast/BlastAlign.cgi Disallow: /blast/bl2seq/wblast2.cgi Disallow: /sites Disallow: /portal Disallow: /pmc/utilities/ Disallow: /pmc/issues/ Disallow: /pmc/articles/ Disallow: /pmc/ivip/ Disallow: /pmc/articlerender.fcgi Disallow: /pmc/pagerender.fcgi Disallow: /pmc/picrender.fcgi Disallow: /pmc/tocrender.fcgi Disallow: /pmc/*report= Disallow: /pmc/*page= Disallow: /pmc/articles/*/citedby/ Sitemap: http://www.ncbi.nih.gov/pmc/site_map_index.xml Sitemap: http://www.ncbi.nlm.nih.gov/escan/?p=books/sm_books.cgi Sitemap: http://www.ncbi.nlm.nih.gov/escan/?p=journals/gdump_journals_main.xml Sitemap: http://www.ncbi.nlm.nih.gov/escan/?p=mesh/gdump_mesh_main.xml Sitemap: http://www.ncbi.nlm.nih.gov/escan/?p=ncbi/sitemap.xml Sitemap: http://www.ncbi.nlm.nih.gov/escan/?p=pubmed/gdump_pubmed_main.xml Sitemap: http://www.pubmedcentral.nih.gov/site_map_index.xml