User-agent: *
Disallow: /*&cp=
Disallow: /comment-page-*
Disallow: /latest-rob-pattinson-new-moon-vic-6463/
Disallow: admin.php
Disallow: /scripts/
Disallow: z-list
Disallow: config
Disallow: header
Disallow: footer
Disallow: pntables
Disallow: referer
Disallow: /ebay/
Disallow: /ebay2/
Disallow: /amazon/
Disallow: /includes/
Disallow: /modules/NS-
Disallow: /pnadodb/
Disallow: /themes/
Disallow: /archives/backup/
Disallow: /cgi-bin/
Disallow: /advancediron/
Disallow: /mt-test/
Disallow: /test/
Disallow: /astracker/
Disallow: /astracker.old/
Disallow: /archives.bak/
Disallow: /mediamayhem/
Disallow: /slideshow/
Disallow: /wp-content/cache/
Disallow: /movcon/
Disallow: /movfiles/
Disallow: /bots/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /demotag/

Disallow: /movies/
Disallow: /advertising/

Disallow: /america-then-now-vic-13391/

#Remove author pages
Disallow: /author/brusimm/
Disallow: /author/kofi/
Disallow: /author/kane/
Disallow: /author/vic/
Disallow: /author/heath/
Disallow: /author/niall/
Disallow: /author/filmnerdjamie/
Disallow: /author/brian/
Disallow: kane_adrian.html
Disallow: bruce_simmons.html
Disallow: vic_holtreman.html
Disallow: heath_mcknight.html
Disallow: jamie_williams.html
Disallow: niall_browne.html
Disallow: larae_meadows.html
Disallow: sybil_vasche.html
Disallow: mike_reiter.html
Disallow: brian_rentschler.html
Disallow: miranda_holtreman.html

# Allow Google Adwords spider
User-agent: Mediapartners-Google* 
Disallow: /index2.html

#Limit Yahoo Crawl due to big bandwidth usage and lousy ranking
User-agent: Slurp
Crawl-delay: 240
Disallow: /store/

# Disallow indexing of specific file extensions 
User-agent: * 

User-agent: * 
Disallow: /*.zip$ 
Disallow: /*.doc$ 

User-agent: * 
Disallow: /*.exe$ 

User-agent: * 
Disallow: /*.pdf$ 

# Disallow Collectors and Spam 
User-agent: atSpider 
Disallow: / 

User-agent: cherrypicker 
Disallow: / 

User-agent: DSurf 
Disallow: / 

User-agent: EliteSys Entry 
Disallow: / 

User-agent: EmailCollector 
Disallow: / 

User-agent: EmailSiphon 
Disallow: / 

User-agent: EmailWolf 
Disallow: / 

User-agent: Mail Sweeper 
Disallow: / 

User-agent: munky 
Disallow: / 

User-agent: Roverbot 
Disallow: / 

User-agent: WebEmailExtrac 
Disallow: / 

# Disallow Offline Browsers 
User-agent: xGet 
Disallow: / 

User-agent: wGet 
Disallow: / 

User-agent: WebWalk 
Disallow: / 

User-agent: webvac 
Disallow: / 

User-agent: WebReaper 
Disallow: / 

User-agent: WebMirror 
Disallow: / 

User-agent: WebFetcher 
Disallow: / 

User-agent: WebCopy 
Disallow: / 

User-agent: webcopier 
Disallow: / 

User-agent: WebCatcher 
Disallow: / 

User-agent: WebBandit 
Disallow: / 

User-agent: w3mir 
Disallow: / 

User-agent: vobsub 
Disallow: / 

User-agent: Templeton 
Disallow: / 

User-agent: ssearcher100 
Disallow: / 

User-agent: SpiderBot 
Disallow: / 

User-agent: Shai'Hulud 
Disallow: / 

User-agent: PBWF 
Disallow: / 

User-agent: LightningDownload 
Disallow: / 

User-agent: KDD Exploror 
Disallow: / 

User-agent: Internet Explore 
Disallow: / 

User-agent: InfoSpiders 
Disallow: / 

User-agent: httrack 
Disallow: / 

User-agent: HavIndex 
Disallow: / 

User-agent: GetUrl 
Disallow: / 

User-agent: GetBot 
Disallow: / 

User-agent: ESIRover 
Disallow: / 

User-agent: Download Wonder 
Disallow: / 

User-agent: Collage 
Disallow: / 

User-agent: Mozilla/2.0 (compatible; MS FrontPage 4.0) 
Disallow: /