The robots.txt file is just a simple text file as the file extension suggests. It's created using a simple text editor like notepad or wordpad.
|
User-Agent: Black Hole
Disallow: /
|
|
|
|
User-Agent: Titan
Disallow: /
|
|
|
|
User-Agent: WebStripper
Disallow: /
|
|
|
|
User-Agent: NetMechanic
Disallow: /
|
|
|
|
User-Agent: CherryPicker
Disallow: /
|
|
|
|
User-Agent: EmailSiphon
Disallow: /
|
|
|
|
User-Agent: WebBandit
Disallow: /
|
|
|
|
User-Agent: EmailWolf
Disallow: /
|
|
|
|
User-Agent: ExtractorPro
Disallow: /
|
|
|
|
User-Agent: CopyRightCheck
Disallow: /
|
|
|
|
User-Agent: Crescent
Disallow: /
|
|
|
|
User-Agent:NICErsPRO
Disallow: /
|
|
|
|
User-Agent: Wget
Disallow: /
|
|
|
|
User-Agent:SiteSnagger
Disallow: /
|
|
|
|
User-Agent:ProWebWalker
Disallow: /
|
|
|
|
User-Agent: CheeseBot
Disallow: /
|
|
|
|
User-Agent: mozilla/4
Disallow: /
|
|
|
|
User-Agent: mozilla/5
Disallow: /
|
|
|
|
User-Agent:Mozilla/4.0 (compatible; MSIE 4.0; Windows NT)
Disallow: /
|
|
|
|
User-Agent:Mozilla/4.0 (compatible; MSIE 4.0; Windows 95)
Disallow: /
|
|
|
|
User-Agent:Mozilla/4.0
(compatible; MSIE 4.0; Windows 9
Disallow: /
|
|
|
|
User-Agent: ia_archiver
Disallow: /
|
|
|
|
User-Agent: ia_archiver/1.6
Disallow: /
|
|
|
|
User-Agent: Alexibot
Disallow: /
|
|
|
|
User-Agent: Teleport
Disallow: /
|
|
|
|
User-Agent: TeleportPro
Disallow: /
|
|
|
|
User-Agent: Wget
Disallow: /
|
|
|
|
User-Agent: MIIxpc
Disallow: /
|
|
|
|
User-Agent: Telesoft
Disallow: /
|
|
|
|
User-Agent: Website Quester
Disallow: /
|
|
|
|
User-Agent: WebZip
Disallow: /
|
|
|
|
User-Agent: moget/2.1
Disallow: /
|
|
|
|
User-Agent: WebStripper
Disallow: /
|
|
|
|
User-Agent: WebSauger
Disallow: /
|
|
|
|
User-Agent: WebCopier
Disallow: /
|
|
|
|
User-Agent: NetAnts
Disallow: /
|
|
|
|
User-Agent: Mister PiX
Disallow: /
|
|
|
|
User-Agent: WebAuto
Disallow: /
|
|
|
|
User-Agent: TheNomad
Disallow: /
|
|
|
|
User-Agent: WWW-Collector-E
Disallow: /
|
|
|
|
User-Agent: RMA
Disallow: /
|
|
|
|
User-Agent:libWeb/clsHTTP
Disallow: /
|
|
|
|
User-Agent: asterias
Disallow: /
|
|
|
|
User-Agent: turingos
Disallow: /
|
|
|
|
User-Agent: spanner
Disallow: /
|
|
|
|
User-Agent: InfoNaviRobot
Disallow: /
|
|
|
|
User-Agent: Harvest/1.5
Disallow: /
|
|
|
|
User-Agent: ExtractorPro
Disallow: /
|
|
|
|
User-Agent: Bullseye/1.0
Disallow: /
|
|
|
|
User-Agent:Mozilla/4.0 (compatible; BullsEye; Windows 95)
Disallow: /
|
|
|
|
User-Agent: Crescent Internet ToolPak HTTPOLE Control v.1.0
Disallow: /
|
|
|
|
User-Agent: CherryPickerSE/1.0
Disallow: /
|
|
|
|
User-Agent: CherryPickerElite/1.0
Disallow: /
|
|
|
|
User-Agent: WebBandit/3.50
Disallow: /
|
|
|
|
User-Agent: NICErsPRO
Disallow: /
|
|
|
|
User-Agent: Microsoft URL Control - 5.01.4511
Disallow: /
|
|
|
|
User-Agent: DittoSpyder
Disallow: /
|
|
|
|
User-Agent: Foobot
Disallow: /
|
|
|
|
User-Agent:WebmasterWorld
ForumBot
Disallow: /
|
|
|
|
User-Agent: SpankBot
Disallow: /
|
|
|
|
User-Agent: BotALot
Disallow: /
|
|
|
|
User-Agent: lwp-trivial/1.34
Disallow: /
|
|
|
|
User-Agent: lwp-trivial
Disallow: /
|
|
|
|
User-Agent: BunnySlippers
Disallow: /
|
|
|
|
User-Agent:Microsoft URL Control - 6.00.8169
Disallow: /
|
|
|
|
User-Agent: URLy Warning
Disallow: /
|
|
|
|
User-Agent: Wget
Disallow: /
|
|
|
|
User-Agent: Wget/1.5.3
Disallow: /
|
|
|
|
User-Agent: LinkWalker
Disallow: /
|
|
|
|
User-Agent:cosmos
Disallow: /
|
|
|
|
User-Agent: moget
Disallow: /
|
|
|
|
User-Agent:hloader
Disallow: /
|
|
|
|
User-Agent: humanlinks
Disallow: /
|
|
|
|
User-Agent: LinkextractorPro
Disallow: /
|
|
|
|
User-Agent: Offline Explorer
Disallow: /
|
|
|
|
User-Agent: Mata Hari
Disallow: /
|
|
|
|
User-Agent: LexiBot
Disallow: /
|
|
|
|
User-Agent: Offline Explorer
Disallow: /
|
|
|
|
User-Agent: Web Image Collector
Disallow: /
|
|
|
|
User-Agent: True_Robot/1.0
Disallow: /
|
|
|
|
User-Agent: True_Robot
Disallow: /
|
|
|
|
User-Agent: BlowFish/1.0
Disallow: /
|
|
|
|
User-Agent: JennyBot
Disallow: /
|
|
|
|
User-Agent: MIIxpc/4.2
Disallow: /
|
|
|
|
User-Agent: BuiltBotTough
Disallow: /
|
|
|
|
User-Agent: ProPowerBot/2.14
Disallow: /
|
|
|
|
User-Agent: BackDoorBot/1.0
Disallow: /
|
|
|
|
User-Agent:toCrawl/UrlDispatcher
Disallow: /
|
|
|
|
User-Agent: WebEnhancer
Disallow: /
|
|
|
|
User-Agent: TightTwatBot
Disallow: /
|
|
|
|
User-Agent: suzuran
Disallow: /
|
|
|
|
User-Agent: VCI WebViewer VCI WebViewer Win32
Disallow: /
|
|
|
|
User-Agent: VCI
Disallow: /
|
|
|
|
User-Agent:Szukacz/1.4
Disallow: /
|
|
|
|
User-Agent: QueryN Metasearch
Disallow: /
|
|
|
|
User-Agent:Openfind data gathere
Disallow: /
|
|
|
|
User-Agent:Openfind
Disallow: /
|
|
|
|
User-Agent: Xenu's Link Sleuth 1.1c
Disallow: /
|
|
|
|
User-Agent:Xenu's
Disallow: /
|
|
|
|
User-Agent:Zeus
Disallow: /
|
|
|
|
User-Agent: RepoMonkey Bait & Tackle/v1.01
Disallow: /
|
|
|
|
User-Agent:RepoMonkey
Disallow: /
|
|
|
|
User-Agent:Zeus 32297 Webster Pro V2.9 Win32
Disallow: /
|
|
|
|
User-Agent: Webster Pro
Disallow: /
|
|
|
|
User-Agent:EroCrawler
Disallow: /
|
|
|
|
User-Agent:LinkScan/8.1a
Disallow: /
|
|
|
|
User-Agent: Kenjin Spider
Disallow: /
|
|
|
|
User-Agent:Keyword Density/0.9
Disallow: /
|
|
|
|
User-Agent:Kenjin Spider
Disallow: /
|
|
|
|
User-Agent:Cegbfeieh
Disallow: /
|
|
|
|
User-Agent:larbin
Disallow: /
|
|
|
|
User-Agent:b2w/0.1
Disallow: /
|
|
|
|
User-Agent: Copernic
Disallow: /
|
|
|
|
User-Agent: psbot
Disallow: /
|
|
|
|
User-Agent:Python-urllib
Disallow: /
|
|
|
|
User-Agent:NetMechanic
Disallow: /
|
|
|
|
User-Agent:URL_Spider_Pro
Disallow: /
|
|
|
|
User-Agent:CherryPicker
Disallow: /
|
|
|
|
User-Agent: EmailCollector
Disallow: /
|
|
|
|
User-Agent: EmailSiphon
Disallow: /
|
|
|
|
User-Agent:WebBandit
Disallow: /
|
|
|
|
User-Agent: EmailWolf
Disallow: /
|
|
|
|
User-Agent:ExtractorPro
Disallow: /
|
|
|
|
User-Agent: CopyRightCheck
Disallow: /
|
|
|
|
User-Agent: Crescent
Disallow: /
|
|
|
|
User-Agent: SiteSnagger
Disallow: /
|
|
|
|
User-Agent:ProWebWalker
Disallow: /
|
|
|
|
User-Agent: CheeseBot
Disallow: /
|
|
|
|
User-Agent:LNSpiderguy
Disallow: /
|
|
|
|
User-Agent: Mozilla
Disallow: /
|
|
|
|
User-Agent: mozilla
Disallow: /
|
|
|
|
User-Agent:mozilla/3
Disallow: /
|
|
|
|
User-Agent:mozilla/4
Disallow: /
|
|
|
|
User-Agent: mozilla/5
Disallow: /
|
|
|
|
User-Agent:WebAuto
Disallow: /
|
|
|
|
User-Agent:TheNomad
Disallow: /
|
|
|
|
User-Agent: WWW-Collector-E
Disallow: /
|
|
|
|
User-Agent:RMA
Disallow: /
|
|
|
|
User-Agent:libWeb/clsHTTP
Disallow: /
|
|
|
|
User-Agent: httplib
Disallow: /
|
|
|
|
User-Agent:turingos
Disallow: /
|
|
|
|
User-Agent: InfoNaviRobot
Disallow: /
|
|
|
|
User-Agent: Harvest/1.5
Disallow: /
|
|
|
|
User-Agent: Crescent Internet
ToolPak HTTP OLE Control v.1.0
Disallow: /
|
|
|
|
User-Agent: CherryPickerSE/1.0
Disallow: /
|
|
|
|
User-Agent:CherryPickerElite/1.0
Disallow: /
|
|
|
|
User-Agent:WebBandit/3.50
Disallow: /
|
|
|
|
User-Agent: NICErsPRO
Disallow: /
|
|
|
|
User-Agent: DittoSpyder
Disallow: /
|
|
|
|
User-Agent:Foobot
Disallow: /
|
|
|
|
User-Agent:BotALot
Disallow: /
|
|
|
|
User-Agent: lwp-trivial/1.34
Disallow: /
|
|
|
|
User-Agent:lwp-trivial
Disallow: /
|
|
|
|
User-Agent: URLy Warning
Disallow: /
|
|
|
|
User-Agent: hloader
Disallow: /
|
|
|
|
User-Agent: humanlinks
Disallow: /
|
|
|
|
User-Agent:LinkextractorPro
Disallow: /
|
|
|
|
User-Agent:Offline Explorer
Disallow: /
|
|
|
|
User-Agent:Mata Hari
Disallow: /
|
|
|
|
User-Agent: LexiBot
Disallow: /
|
|
|
|
User-Agent: Web Image Collector
Disallow: /
|
|
|
|
User-Agent:The Intraformant
Disallow: /
|
|
|
|
User-Agent:True_Robot/1.0
Disallow: /
|
|
|
|
User-Agent: True_Robot
Disallow: /
|
|
|
|
User-Agent: BlowFish/1.0
Disallow: /
|
|
|
|
User-Agent:JennyBot
Disallow: /
|
|
|
|
User-Agent: MIIxpc/4.2
Disallow: /
|
|
|
|
User-Agent: BuiltBotTough
Disallow: /
|
|
|
|
User-Agent:ProPowerBot/2.14
Disallow: /
|
|
|
|
User-Agent: BackDoorBot/1.0
Disallow: /
|
|
|
|
User-Agent:toCrawl/UrlDispatcher
Disallow: /
|
|
|
|
User-Agent:WebEnhancer
Disallow: /
|
|
|
|
User-Agent: suzuran
Disallow: /
|
|
|
|
User-Agent:VCI WebViewer VCI
WebViewer Win32
Disallow: /
|
|
|
|
User-Agent:VCI
Disallow: /
|
|
|
|
User-Agent: Szukacz/1.4
Disallow: /
|
|
|
|
User-Agent: QueryN Metasearch
Disallow: /
|
|
|
|
User-Agent: Openfind data gathere
Disallow: /
|
|
|
|
User-Agent: Openfind
Disallow: /
|
|
|
|
User-Agent:Xenu's Link Sleuth 1.1c
Disallow: /
|
|
|
|
User-Agent:Xenu's
Disallow: /
|
|
|
|
User-Agent: RepoMonkey Bait
& Tackle/v1.01
Disallow: /
|
|
|
|
User-Agent: RepoMonkey
Disallow: /
|
|
|
|
User-Agent:Openbot
Disallow: /
|
|
|
|
User-Agent: URL Control
Disallow: /
|
|
|
|
User-Agent:Zeus Link Scout
Disallow: /
|
|
|
|
User-Agent: Zeus 32297 Webster
Pro V2.9 Win32
Disallow: /
|
|
|
|
User-Agent: EroCrawler
Disallow: /
|
|
|
|
User-Agent:LinkScan/8.1a Unix
Disallow: /
|
|
|
|
User-Agent:Keyword Density/0.9
Disallow: /
|
|
|
|
User-Agent: Kenjin Spider
Disallow: /
|
|
|
|
User-Agent:Iron33/1.0.2
Disallow: /
|
|
|
|
User-Agent: Bookmark search tool
Disallow: /
|
|
|
|
User-Agent: GetRight/4.2
Disallow: /
|
|
|
|
User-Agent:FairAd Client
Disallow: /
|
|
|
|
User-Agent: Gaisbot
Disallow: /
|
|
|
|
User-Agent: Aqua_Products
Disallow: /
|
|
|
|
User-Agent:Radiation Retriever 1.1
Disallow: /
|
|
|
|
User-Agent:WebmasterWorld Extractor
Disallow: /
|
|
|
|
User-Agent: Flaming AttackBot
Disallow: /
|
|
|
|
User-Agent:Oracle Ultra Search
Disallow: /
|
|
|
|
User-Agent:MSIECrawler
Disallow: /
|
|
|
|
User-Agent: PerMan
Disallow: /
|
|
|
|
User-Agent:searchpreview
Disallow: /
|
|
|
|
User-Agent:naver
Disallow: /
|
|
|
|
User-Agent: dumbot
Disallow: /
|
|
|
|
User-Agent:Hatena Antenna
Disallow: /
|
|
|
|
User-Agent:grub-client
Disallow: /
|
|
|
|
User-Agent: grub
Disallow: /
|
|
|
|
User-Agent: larbin
Disallow: /
|
|
|
|
User-Agent: Zeus 32297 Webster
Pro V2.9 Win32
Disallow: /
|
|
|
|
User-Agent: EroCrawler
Disallow: /
|
|
|
|
User-Agent:LinkScan/8.1a Unix
Disallow: /
|
|
|
|
User-Agent:b2w/0.1
Disallow: /
|
|
|
|
User-Agent: Copernic
Disallow: /
|
|
|
|
User-Agent: psbot
Disallow: /
|
|
|
|
User-Agent:Python-urllib
Disallow: /
|
|
|
|
User-Agent: EmailWolf
Disallow: /
|
|
|
|
User-Agent: ExtractorPro
Disallow: /
|
|
|
|
User-Agent:CopyRightCheck
Disallow: /
|
|
|
|
User-Agent: Crescent
Disallow: /
|
|
|
|
User-Agent:SiteSnagger
Disallow: /
|
|
|
|
User-Agent:ProWebWalker
Disallow: /
|
|
|
|
User-Agent: CheeseBot
Disallow: /
|
|
|
|
User-Agent:LNSpiderguy
Disallow: /
|
|
|
|
User-Agent:Mister PiX
Disallow: /
|
|
|
|
User-Agent: WebAuto
Disallow: /
|
|
|
|
User-Agent:TheNomad
Disallow: /
|
|
|
|
User-Agent: WWW-Collector-E
Disallow: /
|
|
|
|
User-Agent: RMA
Disallow: /
|
|
|
|
User-Agent:httplib
Disallow: /
|
|
|
|
User-Agent:turingos
Disallow: /
|
|
|
|
User-Agent: InfoNaviRobot
Disallow: /
|
|
|
|
User-Agent:Harvest/1.5
Disallow: /
|
|
|
|
User-Agent:Bullseye/1.0
Disallow: /
|
|
|
|
User-Agent:Mozilla/4.0 (compatible;
BullsEye; Windows 95)
Disallow: /
|
|
|
|
User-Agent:Crescent Internet
ToolPak HTTP OLE Control v.1.0
Disallow: /
|
|
|
|
User-Agent:CherryPickerSE/1.0
Disallow: /
|
|
|
|
User-Agent:CherryPickerElite/1.0
Disallow: /
|
|
|
|
User-Agent:NICErsPRO
Disallow: /
|
|
|
|
User-Agent:URLy Warning
Disallow: /
|
|
|
|
User-Agent: humanlinks
Disallow: /
|
|
|
|
User-Agent: Web Image Collector
Disallow: /
|
|
|
|
User-Agent:The Intraformant
Disallow: /
|
|
|
|
User-Agent: True_Robot/1.0
Disallow: /
|
|
|
|
User-Agent:True_Robot
Disallow: /
|
|
|
|
User-Agent: BlowFish/1.0
Disallow: /
|
|
|
|
User-Agent: JennyBot
Disallow: /
|
|
|
|
User-Agent: MIIxpc/4.2
Disallow: /
|
|
|
|
User-Agent:BuiltBotTough
Disallow: /
|
|
|
|
User-Agent: ProPowerBot/2.14
Disallow: /
|
|
|
|
User-Agent:BackDoorBot/1.0
Disallow: /
|
|
|
|
User-Agent:toCrawl/UrlDispatcher
Disallow: /
|
|
|
|
User-Agent: WebEnhancer
Disallow: /
|
|
|
|
User-Agent:suzuran
Disallow: /
|
|
|
|
User-Agent:VCI WebViewer VCI
WebViewer Win32
Disallow: /
|
|
|
|
User-Agent: VCI
Disallow: /
|
|
|
|
User-Agent:Szukacz/1.4
Disallow: /
|
|
|
|
User-Agent:QueryN Metasearch
Disallow: /
|
|
|
|
User-Agent: Openfind data gathere
Disallow: /
|
|
|
|
User-Agent:Openfind
Disallow: /
|
|
|
|
User-Agent: Xenu's Link Sleuth 1.1c
Disallow: /
|
|
|
|
User-Agent: Xenu's
Disallow: /
|
|
|
|
User-Agent:Zeus
Disallow: /
|
|
|
|
User-Agent:RepoMonkey Bait &
Tackle/v1.01
Disallow: /
|
|
|
|
User-Agent: RepoMonkey
Disallow: /
|
|
|
|
User-Agent:Microsoft URL Control
Disallow: /
|
|
|
|
User-Agent: Openbot
Disallow: /
|
|
|
|
User-Agent: URL Control
Disallow: /
|
|
|
|
User-Agent:Zeus Link Scout
Disallow: /
|
|
|
|
User-Agent:Zeus 32297 Webster Pro V2.9 Win32
Disallow: /
|
|
|
|
User-Agent: Webster Pro
Disallow: /
|
|
|
|
User-Agent:EroCrawler
Disallow: /
|
|
|
|
User-Agent:LinkScan/8.1a Unix
Disallow: /
|
|
|
|
User-Agent: Keyword Density/0.9
Disallow: /
|
|
|
|
User-Agent:Kenjin Spider
Disallow: /
|
|
|
|
User-Agent: Iron33/1.0.2
Disallow: /
|
|
|
|
User-Agent: Bookmark search tool
Disallow: /
|
|
|
|
User-Agent: GetRight/4.2
Disallow: /
|
|
|
|
User-Agent: FairAd Client
Disallow: /
|
|
|
|
User-Agent: Gaisbot
Disallow: /
|
|
|
|
User-Agent: Aqua_Products
Disallow: /
|
|
|
|
User-Agent: Radiation Retriever 1.1
Disallow: /
|
|
|
|
User-Agent:WebmasterWorld Extractor
Disallow: /
|
|
|
|
User-Agent: Flaming AttackBot
Disallow: /
|
|
|
|
User-Agent:Oracle Ultra Search
Disallow: /
|
|
|
|
User-Agent: MSIECrawler
Disallow: /
|
|
|
|
User-Agent:PerMan
Disallow: /
|
|
|
|
User-Agent: searchpreview
Disallow: /
|
|
|
|
User-Agent: sootle
Disallow: /
|
|
|
|
User-Agent:es
Disallow: /
|
|
|
|
User-Agent:Enterprise_Search/1.0
Disallow: /
|
|
|
|
User-Agent: Enterprise_Search
Disallow: /
|
|
|
