五一快乐，隔离一时爽，一直隔离一直爽

engine/concurrentengine.go

@@ -0,0 +1,76 @@
+package engine
+
+import (
+	"crawl/fetcher"
+	"fmt"
+	"log"
+)
+
+type Scheduler interface {
+	Submit(Request)
+
+	configureWorkChan(chan Request)
+}
+
+type ConcurrentEngine struct {
+	Scheduler Scheduler
+	Workcount int
+}
+
+type SimpleScheduler struct {
+	workerchan chan Request
+}
+
+func (s *SimpleScheduler) Submit(r Request) {
+
+	s.workerchan <- r
+}
+
+func (s *SimpleScheduler) configureWorkChan(c chan Request) {
+
+	s.workerchan = c
+}
+
+func (e *ConcurrentEngine) Run(seeds ...Request) {
+
+	in := make(chan Request)
+	out := make(chan ParseResult)
+
+	for i := 0; i < e.Workcount; i++ {
+
+		CreateWork(in, out)
+	}
+
+	for _, r := range seeds {
+		e.Scheduler.Submit(r)
+	}
+
+}
+
+func CreateWork(in chan Request, out chan ParseResult) {
+
+	go func() {
+		for {
+			request := <-in
+
+			result, err := worker(request)
+
+			if err != nil {
+				continue
+			}
+			out <- result
+		}
+	}()
+
+}
+
+func worker(r Request) (ParseResult, error) {
+	fmt.Printf("Fetching url:%s", r.Url)
+	body, err := fetcher.Fetch(r.Url)
+	if err != nil {
+		log.Printf("Fetching Error:%s", r.Url)
+		return ParseResult{}, err
+	}
+	return r.ParseFunc(body), nil
+
+}

engine/engine.go → engine/simpleengine.go

@@ -6,7 +6,10 @@ import (
 	"log"
 )
 
-func Run(seeds ...Request) {
+type simpleEngine struct {
+}
+
+func (s simpleEngine) Run(seeds ...Request) {
 	var requests []Request
 	for _, e := range seeds {
 		requests = append(requests, e)

main.go

@@ -6,7 +6,11 @@ import (
 )
 
 func main() {
-	engine.Run(engine.Request{
+	e := engine.ConcurrentEngine{
+		&engine.SimpleScheduler{},
+		100,
+	}
+	e.Run(engine.Request{
 		Url:       "https://book.douban.com",
 		ParseFunc: parse.ParseTag,
 	})