异步执行

src/main/java/com/tianya/controller/HouseController.java

@@ -2,37 +2,45 @@
 
 import com.tianya.service.HouseService;
 import com.tianya.util.FileUtils;
+import com.tianya.util.SyncUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.math.NumberUtils;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
 
 import java.util.List;
 
 /**
  * @Auther: Chang
  * @Date: 2018/11/12
  */
-@Controller
+@RestController
 @Slf4j
 public class HouseController {
 
 	@Autowired
 	private HouseService houseService;
 
+	@Autowired
+	private SyncUtils syncUtils;
+
 	@RequestMapping("/")
-	public String index() {
-		return "Hello World";
+	public String index(String url, String page) {
+		log.info("url = {}, page = {}", url, page);
+		syncUtils.submit(() -> getComments(url, page));
+		log.info("已经提交申请，请求后续去查看");
+		return "已经提交申请，请求后续去查看";
 	}
 
 	@RequestMapping(value = "/get/comment")
-	public String getComments() {
+	public String getComments(String url, String page) {
 		// 第一步获取评论
-		List<String> comment = houseService.getComment();
+		List<String> comment = houseService.getComment(url, NumberUtils.toInt(page));
 		// 第二步，转化为md格式
 		List<String> commentMd = houseService.transferStrToMD(comment);
 		// 第三步，写入文件。默认写入D盘中，文件名为house.md
-		FileUtils.writeFile(commentMd);
-		return "ok";
+		FileUtils.writeFile(commentMd, url);
+		return "Hello World";
 	}
 }

src/main/java/com/tianya/service/HouseService.java

@@ -1,13 +1,11 @@
 package com.tianya.service;
 
-import com.tianya.controller.HouseController;
 import com.tianya.util.HttpMethod;
 import lombok.extern.slf4j.Slf4j;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
 
@@ -25,7 +23,7 @@
 @Slf4j
 public class HouseService {
 
-	public static final String PREFIX = "https://bbs.tianya.cn/m/post_author-house-447880-";
+	public static final String PREFIX = "https://bbs.tianya.cn/m/";
 	public static final String SUFFIX = ".shtml";
 
 	/**
@@ -47,12 +45,14 @@ public List<String> transferStrToMD(List<String> comment) {
 	 * 获取评论
 	 * @return
 	 */
-	public List<String> getComment() {
+	public List<String> getComment(String innerUrl, int wholePage) {
 		List<String> res = new ArrayList<>();
 		// 92是固定的，帖子的总数
 		log.info("开始请求天涯帖子.....");
-		for (int i = 1; i <= 167; i++) {
-			String url = getUrl(i);
+		// 修改2731即可
+		int page = (wholePage + 4) / 5;
+		for (int i = 1; i <= page; i++) {
+			String url = getUrl(i, innerUrl);
 			String content = HttpMethod.get(url);
 			getParse(content, res);
 			log.info("请求帖子第 " + i + "  行");
@@ -66,9 +66,9 @@ public List<String> getComment() {
 	 * @param i
 	 * @return
 	 */
-	private String getUrl(int i) {
+	private String getUrl(int i, String url) {
 		int pos = 5 * i - 4;
-		String res = PREFIX + pos + SUFFIX;
+		String res = PREFIX + url + "-" + pos + SUFFIX;
 		return res;
 	}
 

src/main/java/com/tianya/util/FileUtils.java

@@ -6,6 +6,8 @@
 import java.io.File;
 import java.io.FileOutputStream;
 import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
 
 /**
  * @author changwenbo
@@ -18,9 +20,12 @@ public class FileUtils {
 	private static final String SUFFIX = "楼: **==" + "\n";
 
 	/** 写入文件中，转化为PDF文档 */
-	public static void writeFile(List<String> res) {
-		try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(new File("f:\\house.md")))) {
+	public static void writeFile(List<String> res, String url) {
+		String uuid = url + "-" + ThreadLocalRandom.current().nextInt(100, 999);
+		String path = "/home/tomcat/apache-tomcat-8.5.23/workspace/download/" + uuid + ".md";
+		try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(new File(path)))) {
 			int cnt = 1;
+			log.info("开始写入磁盘....uuid = {}", uuid);
 			for (String s : res) {
 				// markdown格式
 				String ss = PREFIX + cnt + SUFFIX + "\n";
@@ -29,7 +34,7 @@ public static void writeFile(List<String> res) {
 				bos.write(s.getBytes());
 				cnt++;
 			}
-			System.out.println("success");
+			log.info("写入磁盘成功...uuid = {}", uuid);
 		} catch (Exception e) {
 			log.error("写入文件出错：" + e.getMessage());
 			e.printStackTrace();

src/main/java/com/tianya/util/HttpMethod.java

@@ -24,7 +24,7 @@ public static String get(String url) {
 	}
 
 	private static HttpConfig getConfig() {
-		String cookies = "JSESSIONID=abcUIPubjzh36dpJGfk_w; __guid=1200512923; __guid2=1200512923; deid=a4f1521763def5a36abdc1b524a1be7e; sso=r=696161994&sid=&wsid=2C275020FCBB2CF94EA1AAF8D1CA21D9; user=w=wenber888&id=137776589&f=1; temp=k=282632486&s=&t=1578569735&b=e18fe0b747008162dfc019ab54d61254&ct=1578569735&et=1581161735; right=web4=n&portal=n; temp4=rm=8346ced28c886d2dc2be32e645d2afab; u_tip=; vip=282632486%3D0; __u_a=v2.3.0; time=ct=1578569874.206; __ptime=1578569874256; ty_msg=1578570149252_137776589_0_0_0_0_0_2_0_0_0_0_0; bbs_msg=1578570149481_137776589_0_0_0_0";
+		String cookies = "Hm_lvt_bc5755e0609123f78d0e816bf7dee255=1590380369; __auc=eeb9a7fd1724a0e962f9fb718b4; __cid=CN; __guid=469014980; __guid2=469014980; deid=57432c15994418404fc637b4418eb34c; sso=r=1609006141&sid=&wsid=9ECDE10F1335A0C76BE47F3815679F80; user=w=wenber888&id=137776589&f=1; temp=k=751567138&s=&t=1590380376&b=0e281318d4992eae9d4b0486a1687b28&ct=1590380376&et=1592972376; right=web4=n&portal=n; temp4=rm=9d0be7aa8e62f78dfa10caaa528ef2da; u_tip=; vip=751567138%3D0; JSESSIONID=abcYhud_K6Iwv_Iis1ljx; __asc=079a9f731724bd526f10e329f0d; __u_a=v2.3.0; time=ct=1590410572.735; __ptime=1590410572786; Hm_lpvt_bc5755e0609123f78d0e816bf7dee255=1590410573; ty_msg=1590410602941_137776589_0_0_0_0_0_0_0_0_0_0_0; bbs_msg=1590410603184_137776589_0_0_0_0";
 		String referer = "https://www.baidu.com";
 		Header[] headers= HttpHeader.custom().cookie(cookies).referer(referer).build();
 		HttpConfig config = HttpConfig.custom().headers(headers);

src/main/java/com/tianya/util/SyncUtils.java

@@ -0,0 +1,20 @@
+package com.tianya.util;
+
+import org.springframework.stereotype.Component;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * @author changwenbo
+ * @date 2020/5/27 14:42
+ */
+@Component
+public class SyncUtils {
+
+	private ExecutorService es = Executors.newCachedThreadPool();
+
+	public void submit(Runnable r) {
+		es.submit(r);
+	}
+}

src/main/resources/application.properties

@@ -0,0 +1 @@
+server.port=8085