🎨 优化多线程下载

优化输出，下载，兼容性
多线程下载存在成功下载一定量图片之后会停止运行，不知道原因

Author: Chenjinyi

Config.php

@@ -39,13 +39,13 @@
 define("SPIDERWAIT_TIME_MIN", "1");//最早等待
 define('SPIDERWAIT_TIME_MAX', "10");//最大等待
 //artatstion执行一次循环，睡一会觉觉（误）
-define("ARTSTATION_SLEEP", "true");
-define("ARTSTATION_SLEEP_TIME_MIN", "10");//最小等待
-define("ARTSTATION_SLEEP_TIME_MAX", "20");//最大等待
+define("ARTSTATION_SLEEP", false);
+define("ARTSTATION_SLEEP_TIME_MIN", "1");//最小等待
+define("ARTSTATION_SLEEP_TIME_MAX", "10");//最大等待
 //bilibili执行一次循环， 就-1s
-define("BILIBILI_SLEEP", "true");
-define("BILIBILI_SLEEP_TIME_MIN", "10");//最小等待
-define("BILIBILI_SLEEP_TIME_MAX", "40");//最大等待
+define("BILIBILI_SLEEP", false);
+define("BILIBILI_SLEEP_TIME_MIN", "1");//最小等待
+define("BILIBILI_SLEEP_TIME_MAX", "10");//最大等待
 
 //数据库链接
 define("SAVE_IMAGES_URL_DATABASE", "false");//是否记录图片链接到数据库
@@ -57,5 +57,5 @@
 
 define("SPIDER_LOG", true);//是否使用爬取记录
 
-define('CURL_DOWN_OPT',false);//是否启用多线程下载
-define("CURL_DOWN_NUM",8);//最多多少个同时下载
+define('CURL_DOWN_OPT', false);//是否启用多线程下载
+define("CURL_DOWN_NUM", 8);//最多多少个同时下载

Src/PublicCore.php

@@ -231,54 +231,58 @@ public function spider_wait($status = true, $min = SPIDERWAIT_TIME_MIN, $max = S
             sleep($num);
         }
     }
-
+    
     /**
      * 一个下载调用其他函数的封装
      * @param $string
      * @param $images_arr
      */
     public function quick_down_img($string, $images_arr, $spider_name, $filename_data = null)
     {
-        if (CURL_DOWN_OPT) { //选择下载模式
-            $this->curl_multi_down_images($string, $images_arr, $spider_name, $filename_data);
+        $dir_path = $this->new_dir_name($string);//生成保存路径
+        if (CURL_DOWN_OPT) { //下载图片
+            $this->curl_multi_down_images($images_arr, $dir_path, $spider_name, $filename_data);
         } else {
-            $dir_path = $this->new_dir_name($string);//生成保存路径
-            $this->image_save($images_arr, $dir_path, $spider_name, $filename_data);//下载图片
-            print_r("文件夹现在有:" . $this->images_number($dir_path) . "张图片");
+            $this->image_save($images_arr, $dir_path, $spider_name, $filename_data);
         }
+        print_r("文件夹现在有:" . $this->images_number($dir_path) . "张图片");
     }
 
     /**
-     * CURL多线程下载　
+     * 多线程图片下载
+     * @param $images_arr
+     * @param $dir_path
+     * @param $spider_name
+     * @param null $filename_data
      */
-    public function curl_multi_down_images($string, $images_arr, $spider_name, $filename_data = null)
+    public function curl_multi_down_images($images_arr, $dir_path, $spider_name, $filename_data = null)
     {
-        $dir_path = $this->new_dir_name($string);//生成保存路径
         for ($c = 1; $c <= count($images_arr); $c += CURL_DOWN_NUM) {//开始循环
-
             $mh = curl_multi_init();//初始化　
             $arr = array_slice($images_arr, $c - 1, CURL_DOWN_NUM);
             $conn = [];
 
-            foreach ($arr as $title => $url) {
+            foreach ($arr as $item) {
+                $title = array_keys($item)[0];
                 if (file_exists($dir_path . DIRECTORY_SEPARATOR . $title)) {//检测是否存在
                     echo "已存在" . PHP_EOL;
                     continue;
                 }
                 $conn[$title] = curl_init();
-                curl_setopt($conn[$title], CURLOPT_URL, $url);
+                curl_setopt($conn[$title], CURLOPT_URL, $item[$title]);
+                curl_setopt($conn[$title], CURLOPT_TIMEOUT, 30);
                 curl_setopt($conn[$title], CURLOPT_RETURNTRANSFER, 1);
                 curl_multi_add_handle($mh, $conn[$title]);
+                echo $title . PHP_EOL;
             }
+
             // 执行批处理句柄
             $active = null;
             do {
                 curl_multi_exec($mh, $active); //执行批处理句柄
             } while ($active > 0); //4
-
             foreach ($conn as $title => $url) {
                 $res[$title] = curl_multi_getcontent($conn[$title]);
-                curl_close($conn['title']);
                 curl_multi_remove_handle($mh, $conn[$title]);//释放资源
             }
 
@@ -288,7 +292,6 @@ public function curl_multi_down_images($string, $images_arr, $spider_name, $file
                 file_put_contents($dir_path . DIRECTORY_SEPARATOR . $title, $item);
                 $this->add_log($spider_name, $title . PHP_EOL, $filename_data);
             }
-            print_r("文件夹现在有:" . $this->images_number($dir_path) . "张图片" . PHP_EOL);
         }
     }
 
@@ -323,4 +326,4 @@ public function bMenu($string, $spiderName)
     }
 
     public $splitLine = "<<<<<<<<<<<<<<<<<<================================>>>>>>>>>>>>>>>>>>";
-}
+}

Src/Spider/Artstation.php

@@ -42,7 +42,7 @@ public function get_img_url($result, $spiderCore)
 //            $images_url = str_replace("medium", "large", $images_url);//生成更高清的图片下载地址
 //            $images_url=str_replace('smaller_square','large',$images_url);
 //            preg_replace("/201.[0-9]{3,}/",'',$images_url);
-            $rep = substr_replace($images_url,'large',63,29);
+            $images_url = substr_replace($images_url,'large',63,29);
             $file_name = $value->title . "-" . $value->cover_asset_id;//生成图片名
             $file_name = $spiderCore->image_url_format($images_url, $file_name);
             array_push($images_arr, [$file_name => $images_url]);
@@ -62,12 +62,14 @@ public function index_spider_core($spiderCore, $spider_name, $parm)
     {
         $posts_num = $spiderCore->user_input("请输入爬取页数(1页=50个作品)(默认为：1):", 1);
         for ($start_num = 1; $start_num <= $posts_num; $start_num++) {
-            $url = "https://www.artstation.com/projects.json?page=" . $start_num . $parm;
+            $url = "https://www.artstation.com/projects.json?medium=digital2d&page=" . $start_num . $parm;
             $result = $spiderCore->curl_get($url, $this->userAgent);
             $result = json_decode($result);
             $images_arr = $this->get_img_url($result, $spiderCore);
-
+//            var_dump($images_arr);
+//            die();
             $spiderCore->quick_down_img($this->spider_name . "-" . $spider_name, $images_arr, "Artstation");
+//            $spiderCore->curl_multi_down_images($spider_name, $images_arr, "Artstation");
             $spiderCore->spider_wait(ARTSTATION_SLEEP, ARTSTATION_SLEEP_TIME_MIN, ARTSTATION_SLEEP_TIME_MAX);
         }