modify README.md

Author: hujunxianligong

README.md

@@ -9,15 +9,17 @@ WebCollector is an open source web crawler framework based on Java.It provides
 
 ## Installation
 
+<!--
 ### Using Maven
 
 ```xml
 <dependency>
     <groupId>cn.edu.hfut.dmic.webcollector</groupId>
     <artifactId>WebCollector</artifactId>
-    <version>2.72-beta</version>
+    <version>2.73-alpha</version>
 </dependency>
 ```
+-->
 
 ### Without Maven
 WebCollector jars are available on the [HomePage](https://github.com/CrawlScript/WebCollector).
@@ -28,22 +30,33 @@ WebCollector jars are available on the [HomePage](https://github.com/CrawlScript
 
 ## Example Index
 
+Annotation versions are named with `DemoAnnotatedxxxxxx.java`.
+
 ### Basic
 
-+ [DemoAutoNewsCrawler.java](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAutoNewsCrawler.java) ([Annotation Version](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedAutoNewsCrawler.java))
++ [DemoAutoNewsCrawler.java](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAutoNewsCrawler.java) | [DemoAnnotatedAutoNewsCrawler.java](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedAutoNewsCrawler.java)
 + [DemoManualNewsCrawler.java](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoManualNewsCrawler.java)
 
 
-### MetaData
-
-+ [DemoBingCrawler.java](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoBingCrawler.java)([Annotation Version](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedBingCrawler.java))
+### CrawlDatum and MetaData
 
++ [DemoMetaCrawler.java](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoMetaCrawler.java)
++ [DemoAnnotatedMatchTypeCrawler.java](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedMatchTypeCrawler.java)
++ [DemoAnnotatedDepthCrawler.java](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedDepthCrawler.java)
++ [DemoBingCrawler.java](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoBingCrawler.java) | [DemoAnnotatedBingCrawler.java](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedBingCrawler.java)
++ [DemoAnnotatedDepthCrawler.java](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedDepthCrawler.java)
 
 ### Http Request
 
-+ [DemoBingCrawler.java](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoBingCrawler.java)([Annotation Version](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedBingCrawler.java))
++ [DemoRedirectCrawler.java](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoRedirectCrawler.java)  | [DemoAnnotatedRedirectCrawler.java](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedRedirectCrawler.java)
++ [DemoPostCrawler.java](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoPostCrawler)
 + [AbuyunDynamicProxyRequester.java](src/main/java/cn/edu/hfut/dmic/webcollector/example/AbuyunDynamicProxyRequester.java)
 
+### NextFilter
+
++ [DemoNextFilter.java](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoNextFilter.java)
++ [DemoHashSetNextFilter.java](src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoHashSetNextFilter.java)
+
 
 
 

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoDepthCrawler.java renamed to src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedDepthCrawler.java

@@ -29,9 +29,9 @@
  *
  * @author hu
  */
-public class DemoDepthCrawler extends BreadthCrawler {
+public class DemoAnnotatedDepthCrawler extends BreadthCrawler {
 
-    public DemoDepthCrawler(String crawlPath, boolean autoParse) {
+    public DemoAnnotatedDepthCrawler(String crawlPath, boolean autoParse) {
         super(crawlPath, autoParse);
 
         for (int i = 1; i <= 5; i++) {
@@ -71,7 +71,7 @@ public void afterParse(Page page, CrawlDatums next) {
 
 
     public static void main(String[] args) throws Exception {
-        DemoDepthCrawler crawler = new DemoDepthCrawler("crawl", true);
+        DemoAnnotatedDepthCrawler crawler = new DemoAnnotatedDepthCrawler("crawl", true);
         crawler.getConf().setTopN(5);
         crawler.start(3);
     }

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedRedirectCrawler.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2015 hu
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ */
+package cn.edu.hfut.dmic.webcollector.example;
+
+import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
+import cn.edu.hfut.dmic.webcollector.model.Page;
+import cn.edu.hfut.dmic.webcollector.plugin.ram.RamCrawler;
+
+import java.net.URLEncoder;
+
+/**
+ * Handle 301/302 redirection
+ * @author hu
+ */
+public class DemoAnnotatedRedirectCrawler extends RamCrawler {
+
+    public DemoAnnotatedRedirectCrawler(String keyword, int maxPageNum) throws Exception {
+        for (int pageNum = 1; pageNum <= maxPageNum; pageNum++) {
+            String url = createBingUrl(keyword, pageNum);
+            addSeedAndReturn(url);
+        }
+    }
+
+    // 如果遇到301或者302，手动跳转（将任务加到next中）
+    // 并且复制任务的meta
+    @MatchCode(codes = {301, 302})
+    public void visitRedirect(Page page, CrawlDatums next){
+        next.addAndReturn(page.location()).meta(page.meta());
+    }
+
+
+    @Override
+    public void visit(Page page, CrawlDatums next) {
+        System.out.println("this page is not redirected: " + page.url());
+    }
+
+    public static void main(String[] args) throws Exception {
+        DemoAnnotatedRedirectCrawler crawler = new DemoAnnotatedRedirectCrawler("网络爬虫", 3);
+        crawler.start();
+    }
+
+    /**
+     * 根据关键词和页号拼接Bing搜索对应的URL
+     * @param keyword 关键词
+     * @param pageNum 页号
+     * @return 对应的URL
+     * @throws Exception 异常 
+     */
+    public static String createBingUrl(String keyword, int pageNum) throws Exception {
+        int first = pageNum * 10 - 9;
+        keyword = URLEncoder.encode(keyword, "utf-8");
+        return String.format("http://cn.bing.com/search?q=%s&first=%s", keyword, first);
+    }
+
+}

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoRedirectCrawler.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 2015 hu
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ */
+package cn.edu.hfut.dmic.webcollector.example;
+
+import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
+import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
+import cn.edu.hfut.dmic.webcollector.model.Page;
+import cn.edu.hfut.dmic.webcollector.plugin.ram.RamCrawler;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+import java.net.URLEncoder;
+
+/**
+ * Handle 301/302 redirection
+ * @author hu
+ */
+public class DemoRedirectCrawler extends RamCrawler {
+
+    public DemoRedirectCrawler(String keyword, int maxPageNum) throws Exception {
+        for (int pageNum = 1; pageNum <= maxPageNum; pageNum++) {
+            String url = createBingUrl(keyword, pageNum);
+            addSeedAndReturn(url);
+        }
+    }
+
+    @Override
+    public void visit(Page page, CrawlDatums next) {
+        // 如果遇到301或者302，手动跳转（将任务加到next中）
+        // 并且复制任务的meta
+        if(page.code() == 301 || page.code() == 302){
+            next.addAndReturn(page.location()).meta(page.meta());
+            return;
+        }
+        System.out.println("this page is not redirected: " + page.url());
+    }
+
+    public static void main(String[] args) throws Exception {
+        DemoRedirectCrawler crawler = new DemoRedirectCrawler("网络爬虫", 3);
+        crawler.start();
+    }
+
+    /**
+     * 根据关键词和页号拼接Bing搜索对应的URL
+     * @param keyword 关键词
+     * @param pageNum 页号
+     * @return 对应的URL
+     * @throws Exception 异常 
+     */
+    public static String createBingUrl(String keyword, int pageNum) throws Exception {
+        int first = pageNum * 10 - 9;
+        keyword = URLEncoder.encode(keyword, "utf-8");
+        return String.format("http://cn.bing.com/search?q=%s&first=%s", keyword, first);
+    }
+
+}