1/*2 * Licensed to the Apache Software Foundation (ASF) under one or more3 * contributor license agreements. See the NOTICE file distributed with4 * this work for additional information regarding copyright ownership.5 * The ASF licenses this file to You under the Apache License, Version 2.06 * (the "License"); you may not use this file except in compliance with7 * the License. You may obtain a copy of the License at8 *9 * http://www.apache.org/licenses/LICENSE-2.010 *11 * Unless required by applicable law or agreed to in writing, software12 * distributed under the License is distributed on an "AS IS" BASIS,13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.14 * See the License for the specific language governing permissions and15 * limitations under the License.16 */1718package org.apache.any23.plugin.crawler;
1920import edu.uci.ics.crawler4j.crawler.Page;
21import org.apache.any23.Any23OnlineTestBase;
22import org.junit.Assert;
23import org.junit.Test;
24import org.slf4j.Logger;
25import org.slf4j.LoggerFactory;
2627import java.io.File;
28import java.net.URL;
29import java.util.HashSet;
30import java.util.Set;
31import java.util.Iterator;
3233/**34 * Test case for {@link SiteCrawler}.35 *36 * @author Michele Mostarda (mostarda@fbk.eu)37 */38publicclassSiteCrawlerTestextendsAny23OnlineTestBase {
3940publicstaticfinal Logger logger = LoggerFactory.getLogger(SiteCrawlerTest.class);
4142/**43 * Tests the main crawler use case.44 *45 * @throws Exception if there is an error asserting test data46 */47 @Test
48publicvoid testSiteCrawling() throws Exception {
49 assumeOnlineAllowed();
5051 File tmpFile = File.createTempFile("site-crawler-test", ".storage");
52 tmpFile.delete();
5354final SiteCrawler controller = new SiteCrawler(tmpFile);
55 controller.setMaxPages(100);
56 logger.info("Crawler4j: Setting max num of pages to: " + controller.getMaxPages());
57 controller.setPolitenessDelay(500);
58 logger.info("Crawler4j: Setting Politeness delay to: " + controller.getPolitenessDelay() + "ms");
5960final Set<String> distinctPages = new HashSet<String>();
61 controller.addListener(new CrawlerListener() {
62 @Override
63publicvoid visitedPage(Page page) {
64 distinctPages.add( page.getWebURL().getURL() );
65 Iterator<String> it = distinctPages.iterator();
66while (it.hasNext()) {
67 logger.info("Crawler4j: Fetching page - " + it.next());
68 }
69 }
70 });
7172 controller.start( new URL("http://any23.apache.org/"), false);
7374synchronized (this) {
75this.wait(15 * 1000);
76 }
77 controller.stop();
7879 logger.info("Distinct pages: " + distinctPages.size());
80 Assert.assertTrue("Expected some page crawled.", distinctPages.size() > 0);
81 }
8283 }