<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0"
	xmlns:content="http://purl.org/rss/1.0/modules/content/"
	xmlns:wfw="http://wellformedweb.org/CommentAPI/"
	xmlns:dc="http://purl.org/dc/elements/1.1/"
	xmlns:atom="http://www.w3.org/2005/Atom"
	xmlns:sy="http://purl.org/rss/1.0/modules/syndication/"
	xmlns:slash="http://purl.org/rss/1.0/modules/slash/"
	>

<channel>
	<title>HTTP Blog &#187; Search Results  &#187;  robots.txt</title>
	<atom:link href="http://www.httpblog.com/?s=robots.txt&#038;feed=rss2" rel="self" type="application/rss+xml" />
	<link>http://www.httpblog.com</link>
	<description>Blog about content delivery, web servers, dedicated servers and hosting industry.</description>
	<lastBuildDate>Tue, 17 Aug 2010 16:22:45 +0000</lastBuildDate>
	<language>en</language>
	<sy:updatePeriod>hourly</sy:updatePeriod>
	<sy:updateFrequency>1</sy:updateFrequency>
	<generator>http://wordpress.org/?v=3.0.1</generator>
		<item>
		<title>Slowing down MSN Bot</title>
		<link>http://www.httpblog.com/slowing-down-msn-bot/</link>
		<comments>http://www.httpblog.com/slowing-down-msn-bot/#comments</comments>
		<pubDate>Sun, 03 Dec 2006 06:01:04 +0000</pubDate>
		<dc:creator>http guy</dc:creator>
				<category><![CDATA[Search engines]]></category>

		<guid isPermaLink="false">http://www.httpblog.com/31</guid>
		<description><![CDATA[Sorry, almost forgot the previous night when I was posting about slowing down Yahoo Slurp, that the same technique applies to MSNBot as well. The only difference is a different name (bot name) User-agent definition. User-Agent: msnbot Crawl-Delay: 10 The above 2 lines in robots.txt file will instruct MSN Bot to delay new request by [...]]]></description>
		<wfw:commentRss>http://www.httpblog.com/slowing-down-msn-bot/feed/</wfw:commentRss>
		<slash:comments>0</slash:comments>
		</item>
		<item>
		<title>Slowing down Yahoo Slurp bot</title>
		<link>http://www.httpblog.com/slowing-down-yahoo-slurp-bot/</link>
		<comments>http://www.httpblog.com/slowing-down-yahoo-slurp-bot/#comments</comments>
		<pubDate>Sat, 02 Dec 2006 04:02:18 +0000</pubDate>
		<dc:creator>http guy</dc:creator>
				<category><![CDATA[Search engines]]></category>

		<guid isPermaLink="false">http://www.httpblog.com/30</guid>
		<description><![CDATA[Some people are screaming that Yahoo Slurp is overloading their web site and eating too much resources, including bandwidth. In order to back off Slurp bot with custom delay between each requests (frequency), you can use robots.txt file and place it in your web site root directory, for example http://www.httpblog.com/robots.txt The following robots.txt setup will [...]]]></description>
		<wfw:commentRss>http://www.httpblog.com/slowing-down-yahoo-slurp-bot/feed/</wfw:commentRss>
		<slash:comments>0</slash:comments>
		</item>
	</channel>
</rss>
