Skip to content

Commit

Permalink
✨ Introducing new features. 支持分布式调度,基于xxl-job
Browse files Browse the repository at this point in the history
  • Loading branch information
lishangbu committed Sep 14, 2020
1 parent 7870db1 commit e4cba84
Show file tree
Hide file tree
Showing 21 changed files with 487 additions and 9 deletions.
6 changes: 4 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ Spring Cloud | Hoxton.SR8
Spring Cloud Alibaba | 2.2.1.RELEASE
Spring Security OAuth2 | 2.3.6
Mybatis Plus | 3.4.0
hutool | 5.4.0
hutool | 5.4.1
Avue | 2.6.15


Expand All @@ -57,6 +57,7 @@ pig
├── pig-common-core -- tool core package
├── pig-common-datasource -- dynamic data source package
├── pig-common-log -- Log service package
├── pig-common-job -- xxl-job expand
├── pig-common-mybatis -- mybatis expand
├── pig-common-security -- security tools
├── pig-common-swagger -- api documentation
Expand All @@ -69,7 +70,8 @@ pig
└── pig-visual
└── pig-monitor -- spring boot admin[5001]
├── pig-codegen -- graphical code generation[5002]
└── pig-sentinel-dashboard -- sentinel dashboard [5003]
└── pig-sentinel-dashboard -- sentinel dashboard [5003]
└── pig-xxl-job -- distributed task scheduling [5004]
```
#### Open source co-construction

Expand Down
4 changes: 3 additions & 1 deletion README.zh.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ pig
└── pig-common -- 系统公共模块
├── pig-common-core -- 公共工具类核心包
├── pig-common-datasource -- 动态数据源包
├── pig-common-job -- xxl-job 封装
├── pig-common-log -- 日志服务
├── pig-common-mybatis -- mybatis 扩展封装
├── pig-common-security -- 安全工具类
Expand All @@ -67,7 +68,8 @@ pig
└── pig-visual
└── pig-monitor -- 服务监控 [5001]
├── pig-codegen -- 图形化代码生成 [5002]
└── pig-sentinel-dashboard -- 流量高可用 [5003]
└── pig-sentinel-dashboard -- 流量高可用 [5003]
└── pig-xxl-job -- 分布式任务调度 [5004]
```
#### 开源共建

Expand Down
2 changes: 2 additions & 0 deletions db/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,5 @@ COPY ./pig.sql /docker-entrypoint-initdb.d
COPY ./pig_codegen.sql /docker-entrypoint-initdb.d

COPY ./pig_config.sql /docker-entrypoint-initdb.d

COPY ./pig_job.sql /docker-entrypoint-initdb.d
3 changes: 2 additions & 1 deletion db/pig_config.sql
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ CREATE TABLE `config_info` (
`c_schema` text COLLATE utf8_bin,
PRIMARY KEY (`id`),
UNIQUE KEY `uk_configinfo_datagrouptenant` (`data_id`,`group_id`,`tenant_id`)
) ENGINE=InnoDB AUTO_INCREMENT=13 DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='config_info';
) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='config_info';

-- ----------------------------
-- Records of config_info
Expand All @@ -42,6 +42,7 @@ INSERT INTO `config_info` VALUES (3, 'pig-codegen-dev.yml', 'DEFAULT_GROUP', '##
INSERT INTO `config_info` VALUES (4, 'pig-gateway-dev.yml', 'DEFAULT_GROUP', 'spring:\n cloud:\n gateway:\n locator:\n enabled: true\n routes:\n # 认证中心\n - id: pig-auth\n uri: lb://pig-auth\n predicates:\n - Path=/auth/**\n filters:\n # 验证码处理\n - ValidateCodeGatewayFilter\n # 前端密码解密\n - PasswordDecoderFilter\n #UPMS 模块\n - id: pig-upms-biz\n uri: lb://pig-upms-biz\n predicates:\n - Path=/admin/**\n filters:\n # 限流配置\n - name: RequestRateLimiter\n args:\n key-resolver: \'#{@remoteAddrKeyResolver}\'\n redis-rate-limiter.replenishRate: 100\n redis-rate-limiter.burstCapacity: 200\n # 代码生成模块\n - id: pig-codegen\n uri: lb://pig-codegen\n predicates:\n - Path=/gen/**\n\n\nsecurity:\n encode:\n # 前端密码密钥,必须16位\n key: \'thanks,pig4cloud\'\n\n# 不校验验证码终端\nignore:\n clients:\n - test\n', '6dfb01b7636af477135c1bfeba2fd48d', '2019-11-29 16:32:42', '2020-06-23 10:34:28', NULL, '172.17.0.156', '', '', '网关配置', '', '', 'yaml', '');
INSERT INTO `config_info` VALUES (5, 'pig-monitor-dev.yml', 'DEFAULT_GROUP', 'spring:\n # 安全配置\n security:\n user:\n name: ENC(8Hk2ILNJM8UTOuW/Xi75qg==) # pig\n password: ENC(o6cuPFfUevmTbkmBnE67Ow====) # pig\n', '85509c6f8c67c364dc78301896274f26', '2019-11-29 16:33:05', '2019-11-29 16:33:05', NULL, '127.0.0.1', '', '', '监控配置', NULL, NULL, 'yaml', NULL);
INSERT INTO `config_info` VALUES (6, 'pig-upms-biz-dev.yml', 'DEFAULT_GROUP', 'security:\n oauth2:\n client:\n client-id: ENC(imENTO7M8bLO38LFSIxnzw==)\n client-secret: ENC(i3cDFhs26sa2Ucrfz2hnQw==)\n scope: server\n\n# 数据源\nspring:\n datasource:\n type: com.zaxxer.hikari.HikariDataSource\n driver-class-name: com.mysql.cj.jdbc.Driver\n username: root\n password: root\n url: jdbc:mysql://pig-mysql:3306/pig?characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=false&allowMultiQueries=true&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=Asia/Shanghai\n', '3248f7cf9ea2ce40cd41cd664ec32ae0', '2019-11-29 16:52:32', '2020-03-14 16:24:24', NULL, '172.17.0.125', '', '', '统一权限', 'null', 'null', 'yaml', 'null');
INSERT INTO `config_info` VALUES (7, 'pig-xxl-job-dev.yml', 'DEFAULT_GROUP', '# 数据源\nspring:\n datasource:\n type: com.alibaba.druid.pool.DruidDataSource\n druid:\n driver-class-name: com.mysql.cj.jdbc.Driver\n username: root\n password: root\n url: jdbc:mysql://pig-mysql:3306/pig_job?characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=false&allowMultiQueries=true&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=Asia/Shanghai\n\nxxl:\n job:\n admin:\n addresses: http://pig-xxl-job-admin:8080/xxl-job-admin', '13cb7c27b2b9e06e55f8bd16d62ed14f', '2020-09-14 20:40:24', '2020-09-14 20:52:16', NULL, '0:0:0:0:0:0:0:1', '', '', '分布式调度', '', '', 'yaml', '');
COMMIT;

-- ----------------------------
Expand Down
117 changes: 117 additions & 0 deletions db/pig_job.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
DROP DATABASE IF EXISTS `pig_job`;

CREATE DATABASE `pig_job` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;

use `pig_job`;

SET NAMES utf8mb4;

CREATE TABLE `xxl_job_info` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`job_group` int(11) NOT NULL COMMENT '执行器主键ID',
`job_cron` varchar(128) NOT NULL COMMENT '任务执行CRON',
`job_desc` varchar(255) NOT NULL,
`add_time` datetime DEFAULT NULL,
`update_time` datetime DEFAULT NULL,
`author` varchar(64) DEFAULT NULL COMMENT '作者',
`alarm_email` varchar(255) DEFAULT NULL COMMENT '报警邮件',
`executor_route_strategy` varchar(50) DEFAULT NULL COMMENT '执行器路由策略',
`executor_handler` varchar(255) DEFAULT NULL COMMENT '执行器任务handler',
`executor_param` varchar(512) DEFAULT NULL COMMENT '执行器任务参数',
`executor_block_strategy` varchar(50) DEFAULT NULL COMMENT '阻塞处理策略',
`executor_timeout` int(11) NOT NULL DEFAULT '0' COMMENT '任务执行超时时间,单位秒',
`executor_fail_retry_count` int(11) NOT NULL DEFAULT '0' COMMENT '失败重试次数',
`glue_type` varchar(50) NOT NULL COMMENT 'GLUE类型',
`glue_source` mediumtext COMMENT 'GLUE源代码',
`glue_remark` varchar(128) DEFAULT NULL COMMENT 'GLUE备注',
`glue_updatetime` datetime DEFAULT NULL COMMENT 'GLUE更新时间',
`child_jobid` varchar(255) DEFAULT NULL COMMENT '子任务ID,多个逗号分隔',
`trigger_status` tinyint(4) NOT NULL DEFAULT '0' COMMENT '调度状态:0-停止,1-运行',
`trigger_last_time` bigint(13) NOT NULL DEFAULT '0' COMMENT '上次调度时间',
`trigger_next_time` bigint(13) NOT NULL DEFAULT '0' COMMENT '下次调度时间',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE `xxl_job_log` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`job_group` int(11) NOT NULL COMMENT '执行器主键ID',
`job_id` int(11) NOT NULL COMMENT '任务,主键ID',
`executor_address` varchar(255) DEFAULT NULL COMMENT '执行器地址,本次执行的地址',
`executor_handler` varchar(255) DEFAULT NULL COMMENT '执行器任务handler',
`executor_param` varchar(512) DEFAULT NULL COMMENT '执行器任务参数',
`executor_sharding_param` varchar(20) DEFAULT NULL COMMENT '执行器任务分片参数,格式如 1/2',
`executor_fail_retry_count` int(11) NOT NULL DEFAULT '0' COMMENT '失败重试次数',
`trigger_time` datetime DEFAULT NULL COMMENT '调度-时间',
`trigger_code` int(11) NOT NULL COMMENT '调度-结果',
`trigger_msg` text COMMENT '调度-日志',
`handle_time` datetime DEFAULT NULL COMMENT '执行-时间',
`handle_code` int(11) NOT NULL COMMENT '执行-状态',
`handle_msg` text COMMENT '执行-日志',
`alarm_status` tinyint(4) NOT NULL DEFAULT '0' COMMENT '告警状态:0-默认、1-无需告警、2-告警成功、3-告警失败',
PRIMARY KEY (`id`),
KEY `I_trigger_time` (`trigger_time`),
KEY `I_handle_code` (`handle_code`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE `xxl_job_log_report` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`trigger_day` datetime DEFAULT NULL COMMENT '调度-时间',
`running_count` int(11) NOT NULL DEFAULT '0' COMMENT '运行中-日志数量',
`suc_count` int(11) NOT NULL DEFAULT '0' COMMENT '执行成功-日志数量',
`fail_count` int(11) NOT NULL DEFAULT '0' COMMENT '执行失败-日志数量',
PRIMARY KEY (`id`),
UNIQUE KEY `i_trigger_day` (`trigger_day`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE `xxl_job_logglue` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`job_id` int(11) NOT NULL COMMENT '任务,主键ID',
`glue_type` varchar(50) DEFAULT NULL COMMENT 'GLUE类型',
`glue_source` mediumtext COMMENT 'GLUE源代码',
`glue_remark` varchar(128) NOT NULL COMMENT 'GLUE备注',
`add_time` datetime DEFAULT NULL,
`update_time` datetime DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE `xxl_job_registry` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`registry_group` varchar(50) NOT NULL,
`registry_key` varchar(255) NOT NULL,
`registry_value` varchar(255) NOT NULL,
`update_time` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `i_g_k_v` (`registry_group`,`registry_key`,`registry_value`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE `xxl_job_group` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`app_name` varchar(64) NOT NULL COMMENT '执行器AppName',
`title` varchar(12) NOT NULL COMMENT '执行器名称',
`address_type` tinyint(4) NOT NULL DEFAULT '0' COMMENT '执行器地址类型:0=自动注册、1=手动录入',
`address_list` varchar(512) DEFAULT NULL COMMENT '执行器地址列表,多地址逗号分隔',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE `xxl_job_user` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`username` varchar(50) NOT NULL COMMENT '账号',
`password` varchar(50) NOT NULL COMMENT '密码',
`role` tinyint(4) NOT NULL COMMENT '角色:0-普通用户、1-管理员',
`permission` varchar(255) DEFAULT NULL COMMENT '权限:执行器ID列表,多个逗号分割',
PRIMARY KEY (`id`),
UNIQUE KEY `i_username` (`username`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE `xxl_job_lock` (
`lock_name` varchar(50) NOT NULL COMMENT '锁名称',
PRIMARY KEY (`lock_name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;


INSERT INTO `xxl_job_group`(`id`, `app_name`, `title`, `address_type`, `address_list`) VALUES (1, 'xxl-job-executor-sample', '示例执行器', 0, NULL);
INSERT INTO `xxl_job_info`(`id`, `job_group`, `job_cron`, `job_desc`, `add_time`, `update_time`, `author`, `alarm_email`, `executor_route_strategy`, `executor_handler`, `executor_param`, `executor_block_strategy`, `executor_timeout`, `executor_fail_retry_count`, `glue_type`, `glue_source`, `glue_remark`, `glue_updatetime`, `child_jobid`, `trigger_status`, `trigger_last_time`, `trigger_next_time`) VALUES (1, 1, '0 0 0 * * ? *', '测试任务1', '2018-11-03 22:21:31', '2020-04-15 13:58:35', 'lengleng', '', 'ROUND', 'demoJobHandler', '', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2018-11-03 22:21:31', '', 0, 0, 0);
INSERT INTO `xxl_job_user`(`id`, `username`, `password`, `role`, `permission`) VALUES (1, 'admin', 'e10adc3949ba59abbe56e057f20f883e', 1, NULL);
INSERT INTO `xxl_job_lock` ( `lock_name`) VALUES ( 'schedule_lock');

commit;
16 changes: 16 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,3 +81,19 @@ services:
container_name: pig-codegen
hostname: pig-codegen
image: pig-codegen

pig-xxl-job:
build:
context: ./pig-visual/pig-xxl-job
restart: always
image: pig-xxl-job
container_name: pig-xxl-job

pig-xxl-job-admin:
container_name: pig-xxl-job-admin
image: xuxueli/xxl-job-admin:2.2.0
restart: always
environment:
PARAMS: "--server.port=8080 --spring.datasource.url=jdbc:mysql://pig-mysql:3306/pig_job?Unicode=true&characterEncoding=UTF-8 --spring.datasource.username=root --spring.datasource.password=root"
ports:
- 8080:8080
Original file line number Diff line number Diff line change
Expand Up @@ -81,18 +81,17 @@ public Object resolveArgument(MethodParameter parameter, ModelAndViewContainer m
}

List<OrderItem> orderItemList = new ArrayList<>();
Optional.ofNullable(ascs).ifPresent(s -> orderItemList.addAll(
Arrays.stream(s).filter(Objects::isNull).map(this::clear).map(OrderItem::asc).collect(Collectors.toList())));
Optional.ofNullable(descs).ifPresent(s -> orderItemList.addAll(
Arrays.stream(s).filter(Objects::isNull).map(this::clear).map(OrderItem::desc).collect(Collectors.toList())));
Optional.ofNullable(ascs).ifPresent(s -> orderItemList.addAll(Arrays.stream(s).filter(Objects::isNull)
.map(this::clear).map(OrderItem::asc).collect(Collectors.toList())));
Optional.ofNullable(descs).ifPresent(s -> orderItemList.addAll(Arrays.stream(s).filter(Objects::isNull)
.map(this::clear).map(OrderItem::desc).collect(Collectors.toList())));
page.addOrder(orderItemList);

return page;
}

/**
* 参数清理
*
* @param param 参数
* @return String
*/
Expand Down
43 changes: 43 additions & 0 deletions pig-common/pig-common-job/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ /*
~ * Copyright (c) 2019-2020, 冷冷 (wangiegie@gmail.com).
~ * <p>
~ * Licensed under the GNU Lesser General Public License 3.0 (the "License");
~ * you may not use this file except in compliance with the License.
~ * You may obtain a copy of the License at
~ * <p>
~ * https://www.gnu.org/licenses/lgpl.html
~ * <p>
~ * Unless required by applicable law or agreed to in writing, software
~ * distributed under the License is distributed on an "AS IS" BASIS,
~ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ * See the License for the specific language governing permissions and
~ * limitations under the License.
~ */
-->

<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.pig4cloud</groupId>
<artifactId>pig-common</artifactId>
<version>2.8.1</version>
</parent>

<artifactId>pig-common-job</artifactId>
<packaging>jar</packaging>

<description>pig 定时任务,基于xxl-job</description>


<dependencies>
<!--xxl job-->
<dependency>
<groupId>com.xuxueli</groupId>
<artifactId>xxl-job-core</artifactId>
<version>${xxl-job.version}</version>
</dependency>
</dependencies>
</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package com.pig4cloud.pig.common.job;

import com.pig4cloud.pig.common.job.properties.XxlJobProperties;
import com.xxl.job.core.executor.impl.XxlJobSpringExecutor;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;

/**
* xxl-job自动装配
*
* @author lishangbu
* @date 2020/9/14
*/
@Configuration
@EnableAutoConfiguration
@ComponentScan("com.pig4cloud.pig.common.job.properties")
public class XxlJobAutoConfiguration {

@Bean
public XxlJobSpringExecutor xxlJobSpringExecutor(XxlJobProperties xxlJobProperties) {
XxlJobSpringExecutor xxlJobSpringExecutor = new XxlJobSpringExecutor();
xxlJobSpringExecutor.setAdminAddresses(xxlJobProperties.getAdmin().getAddresses());
xxlJobSpringExecutor.setAppname(xxlJobProperties.getExecutor().getAppname());
xxlJobSpringExecutor.setAddress(xxlJobProperties.getExecutor().getAddress());
xxlJobSpringExecutor.setIp(xxlJobProperties.getExecutor().getIp());
xxlJobSpringExecutor.setPort(xxlJobProperties.getExecutor().getPort());
xxlJobSpringExecutor.setAccessToken(xxlJobProperties.getExecutor().getAccessToken());
xxlJobSpringExecutor.setLogPath(xxlJobProperties.getExecutor().getLogPath());
xxlJobSpringExecutor.setLogRetentionDays(xxlJobProperties.getExecutor().getLogRetentionDays());
return xxlJobSpringExecutor;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package com.pig4cloud.pig.common.job.annotation;

import com.pig4cloud.pig.common.job.properties.XxlJobProperties;
import org.springframework.context.annotation.Import;

import java.lang.annotation.*;

/**
* 激活xxl-job配置
*
* @author lishangbu
* @date 2020/9/14
*/
@Target({ ElementType.TYPE })
@Retention(RetentionPolicy.RUNTIME)
@Documented
@Inherited
@Import({ XxlJobProperties.class })
public @interface EnablePigXxlJob {

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package com.pig4cloud.pig.common.job.properties;

import lombok.Data;

/**
* xxl-job管理平台配置
*
* @author lishangbu
* @date 2020/9/14
*/
@Data
public class XxlAdminProperties {

/**
* 调度中心部署跟地址 [选填]:如调度中心集群部署存在多个地址则用逗号分隔。 执行器将会使用该地址进行"执行器心跳注册"和"任务结果回调";为空则关闭自动注册;
*/
private String addresses = "http://127.0.0.1:8080/xxl-job-admin";

}
Loading

0 comments on commit e4cba84

Please sign in to comment.