开通博客赚积分
发布资源赚积分
分类

源码开发语言/平台
当前位置：首页 > 源码/资料 > Internet/网络编程 > 搜索引擎 > 查看源码
RobotsTXTTest.java
资源名称：jspider-src-0.5.0-dev.zip　[点击查看]
上传用户：qing5858
上传日期：2015-10-27
资源大小：6056k
文件大小：20k
源码类别：

搜索引擎

开发平台：

Java

RobotsTXTTest.java：源码内容
  1. package net.javacoding.jspider.functional.specific.robotstxt;
  3. import junit.framework.TestCase;
  4. import net.javacoding.jspider.JSpider;
  5. import net.javacoding.jspider.functional.TestingConstants;
  6. import net.javacoding.jspider.core.util.config.ConfigurationFactory;
  7. import net.javacoding.jspider.core.util.config.JSpiderConfiguration;
  8. import net.javacoding.jspider.mockobjects.OverridingJSpiderConfiguration;
  9. import net.javacoding.jspider.mockobjects.OverridingPropertySet;
  10. import net.javacoding.jspider.mockobjects.plugin.JUnitEventSink;
  12. import java.net.URL;
  14. /**
  15.  * $Id: RobotsTXTTest.java,v 1.4 2003/04/10 16:19:24 vanrogu Exp $
  16.  */
  17. public class RobotsTXTTest extends TestCase {
  19.     protected JUnitEventSink sink;
  20.     protected JSpiderConfiguration config;
  21.     protected JSpiderConfiguration config2;
  23.     /**
  24.      * Public constructor giving a name to the test.
  25.      */
  26.     public RobotsTXTTest ( ) {
  27.         super ( "RobotsTXTTest ");
  28.     }
  30.     /**
  31.      * JUnit's overridden setUp method
  32.      * @throws java.lang.Exception in case something fails during setup
  33.      */
  34.     protected void setUp() throws Exception {
  35.         System.err.println("setUp");
  36.         config = ConfigurationFactory.getConfiguration(ConfigurationFactory.CONFIG_UNITTEST);
  37.         config2 = new OverridingJSpiderConfiguration ( config );
  38.         ((OverridingPropertySet)config2.getJSpiderConfiguration()).setValue("jspider.userAgent", "JSpiderUnitTest");
  39.         ConfigurationFactory.setConfiguration(config);
  40.         sink = JUnitEventSink.getInstance();
  41.     }
  43.     /**
  44.      * JUnit's overridden tearDown method
  45.      * @throws java.lang.Exception in case something fails during tearDown
  46.      */
  47.     protected void tearDown() throws Exception {
  48.         System.err.println("tearDown");
  49.         ConfigurationFactory.cleanConfiguration();
  50.         sink.reset();
  51.     }
  53.     public void testAllowedNormalUserAgent ( ) throws Exception {
  55.         URL url = new URL ( "http://" + TestingConstants.HOST + "/testcases/specific/robotstxt/resource.html" );
  56.         ConfigurationFactory.setConfiguration(config);
  58.         JSpider jspider = new JSpider ( url );
  59.         jspider.start ( );
  61.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStartedEvent.class,1);
  62.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringSummaryEvent.class,1);
  63.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStoppedEvent.class,1);
  65.         testEventCount(net.javacoding.jspider.api.event.site.SiteDiscoveredEvent.class,1);
  66.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTMissingEvent.class,0);
  67.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchedEvent.class,1);
  68.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchErrorEvent.class,0);
  70.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceDiscoveredEvent.class,2);
  71.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchedEvent.class,2);
  72.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchErrorEvent.class,0);
  73.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceForbiddenEvent.class,0);
  74.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForFetchingEvent.class,0);
  75.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForParsingEvent.class,0);
  76.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceReferenceDiscoveredEvent.class,0);
  77.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceParsedEvent.class,1);
  78.     }
  80.     public void testDisallowedResourceNormalUserAgent ( ) throws Exception {
  82.         URL url = new URL ( "http://" + TestingConstants.HOST + "/testcases/specific/robotstxt/disallowedResource1.html" );
  83.         ConfigurationFactory.setConfiguration(config);
  85.         JSpider jspider = new JSpider ( url );
  86.         jspider.start ( );
  88.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStartedEvent.class,1);
  89.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringSummaryEvent.class,1);
  90.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStoppedEvent.class,1);
  92.         testEventCount(net.javacoding.jspider.api.event.site.SiteDiscoveredEvent.class,1);
  93.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTMissingEvent.class,0);
  94.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchedEvent.class,1);
  95.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchErrorEvent.class,0);
  97.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceDiscoveredEvent.class,2);
  98.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchedEvent.class,1);
  99.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchErrorEvent.class,0);
  100.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceForbiddenEvent.class,1);
  101.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForFetchingEvent.class,0);
  102.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForParsingEvent.class,0);
  103.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceReferenceDiscoveredEvent.class,0);
  104.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceParsedEvent.class,0);
  105.     }
  107.     public void testDisallowedFolderNormalUserAgent ( ) throws Exception {
  109.         URL url = new URL ( "http://" + TestingConstants.HOST + "/testcases/specific/robotstxt/disallowedFolder1/resource.html" );
  110.         ConfigurationFactory.setConfiguration(config);
  112.         JSpider jspider = new JSpider ( url );
  113.         jspider.start ( );
  115.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStartedEvent.class,1);
  116.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringSummaryEvent.class,1);
  117.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStoppedEvent.class,1);
  119.         testEventCount(net.javacoding.jspider.api.event.site.SiteDiscoveredEvent.class,1);
  120.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTMissingEvent.class,0);
  121.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchedEvent.class,1);
  122.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchErrorEvent.class,0);
  124.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceDiscoveredEvent.class,2);
  125.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchedEvent.class,1);
  126.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchErrorEvent.class,0);
  127.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceForbiddenEvent.class,1);
  128.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForFetchingEvent.class,0);
  129.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForParsingEvent.class,0);
  130.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceReferenceDiscoveredEvent.class,0);
  131.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceParsedEvent.class,0);
  132.     }
  134.     public void testSometimesAllowedResourceNormalUserAgent ( ) throws Exception {
  136.         URL url = new URL ( "http://" + TestingConstants.HOST + "/testcases/specific/robotstxt/disallowedResource2.html" );
  137.         ConfigurationFactory.setConfiguration(config);
  139.         JSpider jspider = new JSpider ( url );
  140.         jspider.start ( );
  142.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStartedEvent.class,1);
  143.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringSummaryEvent.class,1);
  144.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStoppedEvent.class,1);
  146.         testEventCount(net.javacoding.jspider.api.event.site.SiteDiscoveredEvent.class,1);
  147.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTMissingEvent.class,0);
  148.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchedEvent.class,1);
  149.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchErrorEvent.class,0);
  151.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceDiscoveredEvent.class,2);
  152.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchedEvent.class,2);
  153.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchErrorEvent.class,0);
  154.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceForbiddenEvent.class,0);
  155.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForFetchingEvent.class,0);
  156.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForParsingEvent.class,0);
  157.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceReferenceDiscoveredEvent.class,0);
  158.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceParsedEvent.class,1);
  159.     }
  161.     public void testSometimesAllowedFolderNormalUserAgent ( ) throws Exception {
  163.         URL url = new URL ( "http://" + TestingConstants.HOST + "/testcases/specific/robotstxt/disallowedFolder2/resource.html" );
  164.         ConfigurationFactory.setConfiguration(config);
  166.         JSpider jspider = new JSpider ( url );
  167.         jspider.start ( );
  169.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStartedEvent.class,1);
  170.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringSummaryEvent.class,1);
  171.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStoppedEvent.class,1);
  173.         testEventCount(net.javacoding.jspider.api.event.site.SiteDiscoveredEvent.class,1);
  174.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTMissingEvent.class,0);
  175.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchedEvent.class,1);
  176.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchErrorEvent.class,0);
  178.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceDiscoveredEvent.class,2);
  179.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchedEvent.class,2);
  180.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchErrorEvent.class,0);
  181.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceForbiddenEvent.class,0);
  182.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForFetchingEvent.class,0);
  183.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForParsingEvent.class,0);
  184.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceReferenceDiscoveredEvent.class,0);
  185.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceParsedEvent.class,1);
  186.     }
  194.     public void testAllowedTestUserAgent ( ) throws Exception {
  196.         URL url = new URL ( "http://" + TestingConstants.HOST + "/testcases/specific/robotstxt/resource.html" );
  197.         ConfigurationFactory.setConfiguration(config2);
  199.         JSpider jspider = new JSpider ( url );
  200.         jspider.start ( );
  202.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStartedEvent.class,1);
  203.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringSummaryEvent.class,1);
  204.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStoppedEvent.class,1);
  206.         testEventCount(net.javacoding.jspider.api.event.site.SiteDiscoveredEvent.class,1);
  207.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTMissingEvent.class,0);
  208.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchedEvent.class,1);
  209.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchErrorEvent.class,0);
  211.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceDiscoveredEvent.class,2);
  212.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchedEvent.class,2);
  213.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchErrorEvent.class,0);
  214.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceForbiddenEvent.class,0);
  215.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForFetchingEvent.class,0);
  216.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForParsingEvent.class,0);
  217.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceReferenceDiscoveredEvent.class,0);
  218.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceParsedEvent.class,1);
  219.     }
  221.     public void testDisallowedResourceTestUserAgent ( ) throws Exception {
  223.         URL url = new URL ( "http://" + TestingConstants.HOST + "/testcases/specific/robotstxt/disallowedResource2.html" );
  224.         ConfigurationFactory.setConfiguration(config2);
  226.         JSpider jspider = new JSpider ( url );
  227.         jspider.start ( );
  229.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStartedEvent.class,1);
  230.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringSummaryEvent.class,1);
  231.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStoppedEvent.class,1);
  233.         testEventCount(net.javacoding.jspider.api.event.site.SiteDiscoveredEvent.class,1);
  234.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTMissingEvent.class,0);
  235.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchedEvent.class,1);
  236.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchErrorEvent.class,0);
  238.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceDiscoveredEvent.class,2);
  239.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchedEvent.class,1);
  240.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchErrorEvent.class,0);
  241.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceForbiddenEvent.class,1);
  242.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForFetchingEvent.class,0);
  243.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForParsingEvent.class,0);
  244.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceReferenceDiscoveredEvent.class,0);
  245.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceParsedEvent.class,0);
  246.     }
  248.     public void testDisallowedFolderTestUserAgent ( ) throws Exception {
  250.         URL url = new URL ( "http://" + TestingConstants.HOST + "/testcases/specific/robotstxt/disallowedFolder2/resource.html" );
  251.         ConfigurationFactory.setConfiguration(config2);
  253.         JSpider jspider = new JSpider ( url );
  254.         jspider.start ( );
  256.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStartedEvent.class,1);
  257.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringSummaryEvent.class,1);
  258.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStoppedEvent.class,1);
  260.         testEventCount(net.javacoding.jspider.api.event.site.SiteDiscoveredEvent.class,1);
  261.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTMissingEvent.class,0);
  262.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchedEvent.class,1);
  263.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchErrorEvent.class,0);
  265.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceDiscoveredEvent.class,2);
  266.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchedEvent.class,1);
  267.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchErrorEvent.class,0);
  268.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceForbiddenEvent.class,1);
  269.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForFetchingEvent.class,0);
  270.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForParsingEvent.class,0);
  271.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceReferenceDiscoveredEvent.class,0);
  272.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceParsedEvent.class,0);
  273.     }
  275.     public void testSometimesAllowedResourceTestUserAgent ( ) throws Exception {
  277.         URL url = new URL ( "http://" + TestingConstants.HOST + "/testcases/specific/robotstxt/disallowedResource1.html" );
  278.         ConfigurationFactory.setConfiguration(config2);
  280.         JSpider jspider = new JSpider ( url );
  281.         jspider.start ( );
  283.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStartedEvent.class,1);
  284.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringSummaryEvent.class,1);
  285.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStoppedEvent.class,1);
  287.         testEventCount(net.javacoding.jspider.api.event.site.SiteDiscoveredEvent.class,1);
  288.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTMissingEvent.class,0);
  289.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchedEvent.class,1);
  290.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchErrorEvent.class,0);
  292.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceDiscoveredEvent.class,2);
  293.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchedEvent.class,2);
  294.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchErrorEvent.class,0);
  295.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceForbiddenEvent.class,0);
  296.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForFetchingEvent.class,0);
  297.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForParsingEvent.class,0);
  298.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceReferenceDiscoveredEvent.class,0);
  299.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceParsedEvent.class,1);
  300.     }
  302.     public void testSometimesAllowedFolderTestUserAgent ( ) throws Exception {
  304.         URL url = new URL ( "http://" + TestingConstants.HOST + "/testcases/specific/robotstxt/disallowedFolder1/resource.html" );
  305.         ConfigurationFactory.setConfiguration(config2);
  307.         JSpider jspider = new JSpider ( url );
  308.         jspider.start ( );
  310.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStartedEvent.class,1);
  311.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringSummaryEvent.class,1);
  312.         testEventCount(net.javacoding.jspider.api.event.engine.SpideringStoppedEvent.class,1);
  314.         testEventCount(net.javacoding.jspider.api.event.site.SiteDiscoveredEvent.class,1);
  315.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTMissingEvent.class,0);
  316.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchedEvent.class,1);
  317.         testEventCount(net.javacoding.jspider.api.event.site.RobotsTXTFetchErrorEvent.class,0);
  319.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceDiscoveredEvent.class,2);
  320.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchedEvent.class,2);
  321.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceFetchErrorEvent.class,0);
  322.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceForbiddenEvent.class,0);
  323.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForFetchingEvent.class,0);
  324.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceIgnoredForParsingEvent.class,0);
  325.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceReferenceDiscoveredEvent.class,0);
  326.         testEventCount(net.javacoding.jspider.api.event.resource.ResourceParsedEvent.class,1);
  327.     }
  330.     protected void testEventCount ( Class eventClass, int expectedCount  ) {
  331.         assertEquals(eventClass.getName(), expectedCount, sink.getEventCount(eventClass));
  332.     }
  334. }