crawSinaImageService

Author: Jason-Chen-2017

src/main/kotlin/com/light/saber/api/GankApiBuilder.kt renamed to src/main/kotlin/com/light/saber/api/GankImageApiBuilder.kt

@@ -1,6 +1,6 @@
 package com.light.saber.api
 
-object GankApiBuilder {
+object GankImageApiBuilder {
     fun build(page: Int): String {
         // 此处 api 中的中文需要转义
         return "http://gank.io/api/data/%E7%A6%8F%E5%88%A9/100/${page}"

src/main/kotlin/com/light/saber/api/ImageSearchApiBuilder.kt

@@ -0,0 +1,7 @@
+package com.light.saber.api
+
+object SinaImageApiBuilder {
+    fun build(page: Int): String {
+        return "http://platform.sina.com.cn/slide/album_photo_col?app_key=1985696825&photo_col_id=13&tags=cat&tagmode=any&format=json&page=${page}"
+    }
+}

src/main/kotlin/com/light/saber/api/SinaImageApiBuilder.kt

@@ -0,0 +1,37 @@
+package com.light.saber.controller
+
+import com.light.saber.service.impl.CrawGankImageService
+import com.light.saber.service.impl.CrawSinaImageService
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.stereotype.Controller
+import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RequestMethod
+import org.springframework.web.bind.annotation.ResponseBody
+
+/**
+ * Created by jack on 2017/7/22.
+ */
+
+@Controller
+class CrawImageController {
+
+    @Autowired
+    lateinit var crawGankImageService: CrawGankImageService
+    @Autowired
+    lateinit var crawSinaImageService: CrawSinaImageService
+
+    @RequestMapping(value = ["crawGankImageService"], method = [(RequestMethod.GET)])
+    @ResponseBody
+    fun crawGankImageService(): String {
+        crawGankImageService.doCrawJob()
+        return "crawGankImageService JOB Started"
+    }
+
+    @RequestMapping(value = ["crawSinaImageService"], method = [(RequestMethod.GET)])
+    @ResponseBody
+    fun crawSinaImageService(): String {
+        crawSinaImageService.doCrawJob()
+        return "crawSinaImageService JOB Started"
+    }
+
+}

src/main/kotlin/com/light/saber/controller/CrawController.kt

@@ -2,8 +2,9 @@ package com.light.saber.controller
 
 import com.light.saber.dao.KnowledgeMapper
 import com.light.saber.model.Knowledge
+import com.light.saber.result.Result
 import com.light.saber.service.KnowledgeService
-import com.light.saber.service.TakeKeyWordsService
+import com.light.saber.service.impl.TakeKeyWordsService
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.data.domain.PageRequest
 import org.springframework.data.domain.Sort

src/main/kotlin/com/light/saber/controller/CrawImageController.kt

@@ -1,7 +1,7 @@
 package com.light.saber.controller
 
 import com.light.saber.crawler.*
-import com.light.saber.service.CrawKnowledgeService
+import com.light.saber.service.impl.CrawKnowledgeService
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.web.bind.annotation.GetMapping
 import org.springframework.web.bind.annotation.RestController

src/main/kotlin/com/light/saber/controller/KnowledgeController.kt

@@ -1,3 +1,3 @@
-package com.light.saber.controller
+package com.light.saber.result
 
 data class Result<T>(var data: T, var msg: String, var success: Boolean)

src/main/kotlin/com/light/saber/controller/KnowledgeCrawController.kt

@@ -1,56 +1,6 @@
 package com.light.saber.service
 
-import com.light.saber.api.GankApiBuilder
-import com.light.saber.dao.ImageRepository
-import com.light.saber.model.Image
-import com.light.saber.util.JsonResultProcessor
-import com.light.saber.webclient.CrawlerWebClient
-import kotlinx.coroutines.experimental.CommonPool
-import kotlinx.coroutines.experimental.launch
-import kotlinx.coroutines.experimental.runBlocking
-import org.slf4j.LoggerFactory
-import org.springframework.beans.factory.annotation.Autowired
-import org.springframework.stereotype.Service
-import java.net.URL
-import java.text.SimpleDateFormat
-import java.util.*
-
-@Service
-class CrawImageService {
-    val logger = LoggerFactory.getLogger(CrawImageService::class.java)
-
-    @Autowired
-    lateinit var CrawlerWebClient: CrawlerWebClient
-
-    @Autowired
-    lateinit var imageRepository: ImageRepository
-
-    fun doGankImageCrawJob() = runBlocking {
-        for (page in 1..7) {
-            launch(CommonPool) {
-                saveGankImage(page)
-            }
-        }
-    }
-
-    private fun saveGankImage(page: Int) {
-        val api = GankApiBuilder.build(page)
-        JsonResultProcessor.getGankImageUrls(api).forEach {
-            val url = it
-            if (imageRepository.countByUrl(url) == 0) {
-                val image = Image()
-                image.category = "干货集中营福利 ${SimpleDateFormat("yyyy-MM-dd").format(Date())}"
-                image.url = url
-                image.sourceType = 1
-                image.imageBlob = getByteArray(url)
-                logger.info("image = ${image}")
-                imageRepository.save(image)
-            }
-        }
-    }
-
-    private fun getByteArray(url: String): ByteArray {
-        val urlObj = URL(url)
-        return urlObj.readBytes()
-    }
-}
+interface CrawImageService {
+    fun doCrawJob()
+    fun saveImage(page: Int)
+}

src/main/kotlin/com/light/saber/controller/Result.kt renamed to src/main/kotlin/com/light/saber/result/Result.kt

@@ -5,7 +5,7 @@ import org.springframework.data.domain.Page
 import org.springframework.data.domain.Pageable
 
 interface KnowledgeService {
-    abstract fun page(title: String?, page: Pageable): Page<Knowledge>
+    fun page(title: String?, page: Pageable): Page<Knowledge>
     fun doSaveKnowledge(articleUrl: String, articleTitle: String, articleBody: String?)
 
 }