Skip to content

Commit

Permalink
使用JSON Splitter拆分记录中的JSON字段内容出错, modify for #390
Browse files Browse the repository at this point in the history
  • Loading branch information
baisui1981 committed Nov 5, 2024
1 parent df9ae92 commit 97994ba
Show file tree
Hide file tree
Showing 18 changed files with 213 additions and 107 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
*/
public abstract class Column {

public final static Column NULL = new NullColumn();

private Type type;

private Object rawData;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.alibaba.datax.common.element;

import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.Date;

/**
* @author: 百岁(baisui@qlangtech.com)
* @create: 2024-11-05 12:33
**/
public class NullColumn extends Column {
NullColumn() {
super(null, Type.NULL, 0);
}

@Override
public Long asLong() {
throw new UnsupportedOperationException();
}

@Override
public Double asDouble() {
throw new UnsupportedOperationException();
}

@Override
public String asString() {
throw new UnsupportedOperationException();
}

@Override
public Date asDate() {
throw new UnsupportedOperationException();
}

@Override
public Date asDate(String dateFormat) {
throw new UnsupportedOperationException();
}

@Override
public byte[] asBytes() {
throw new UnsupportedOperationException();
}

@Override
public Boolean asBoolean() {
throw new UnsupportedOperationException();
}

@Override
public BigDecimal asBigDecimal() {
throw new UnsupportedOperationException();
}

@Override
public BigInteger asBigInteger() {
throw new UnsupportedOperationException();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

package com.alibaba.datax.core.job;

import com.qlangtech.tis.plugin.datax.transformer.OutputParameter;
import com.qlangtech.tis.plugin.ds.IColMetaGetter;
import com.qlangtech.tis.plugin.ds.RunningContext;

Expand Down Expand Up @@ -60,11 +61,11 @@ public interface ITransformerBuildInfo {
* @return
*/
List<IColMetaGetter> originColsWithContextParams();
List<IColMetaGetter> tranformerColsWithoutContextParams();
List<OutputParameter> tranformerColsWithoutContextParams();
/**
* 取得执行当前上线文绑定的参数,例如,当前数据库的名称等
*
* @return
*/
<T extends IColMetaGetter> List<IColMetaGetter> overwriteColsWithContextParams(List<T> sourceCols);
<T extends IColMetaGetter> List<OutputParameter> overwriteColsWithContextParams(List<T> sourceCols);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.qlangtech.tis.plugin.datax.transformer;

import com.qlangtech.tis.plugin.ds.DataType;
import com.qlangtech.tis.plugin.ds.IColMetaGetter;

/**
* @author: 百岁(baisui@qlangtech.com)
* @create: 2024-06-22 11:15
**/
public class OutputParameter implements IColMetaGetter {
private final String name;
private final boolean virtual;
private final DataType type;

public static OutputParameter create(String rename, boolean isVirtual, DataType type) {
return new OutputParameter(rename, isVirtual, type);
}

private OutputParameter(String name, boolean virtual, DataType type) {
this.name = name;
this.virtual = virtual;
this.type = type;
}

@Override
public boolean isPk() {
return false;
}

public String getName() {
return this.name;
}

public boolean isVirtual() {
return this.virtual;
}

public DataType getType() {
return type;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@
package com.qlangtech.tis.maven.plugins.tpi;


import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.SetUtils;
import org.apache.commons.io.IOUtils;
import org.apache.maven.artifact.Artifact;
import org.apache.maven.artifact.resolver.filter.ArtifactFilter;
Expand Down Expand Up @@ -54,7 +52,6 @@
import java.io.Writer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
Expand Down Expand Up @@ -210,7 +207,7 @@ public abstract class AbstractHpiMojo extends AbstractJenkinsMojo {
@Parameter
private String dependentWarExcludes;

@Parameter
@Parameter(defaultValue = "${maven-tpi-plugin.classpathDependentExcludes}")
private String classpathDependentExcludes;

/**
Expand Down Expand Up @@ -477,8 +474,7 @@ public void buildWebapp(MavenProject project, File webappDirectory) throws MojoE
// baisui add for specific dependency filter
ArtifactFilter dependencyFilter = null;
if (this.classpathDependentExcludes != null) {
List<String> excludedDependencies = Arrays.asList(StringUtils.split(this.classpathDependentExcludes, ","));
dependencyFilter = new PatternIncludesArtifactFilter(excludedDependencies);
dependencyFilter = createClasspathDependencyExcludesFilter();
}
File libDirectory = new File(webappDirectory, WEB_INF + "/lib");
File tldDirectory = new File(webappDirectory, WEB_INF + "/tld");
Expand Down Expand Up @@ -573,6 +569,14 @@ public void buildWebapp(MavenProject project, File webappDirectory) throws MojoE
}
}

protected ArtifactFilter createClasspathDependencyExcludesFilter() {
if (StringUtils.isEmpty(this.classpathDependentExcludes)) {
throw new IllegalStateException("property classpathDependentExcludes can not be null");
}
List<String> excludedDependencies = Arrays.asList(StringUtils.split(this.classpathDependentExcludes, ","));
return new PatternIncludesArtifactFilter(excludedDependencies);
}

/**
* Searches a set of artifacts for duplicate filenames and returns a list of duplicates.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import com.qlangtech.tis.manage.common.Config;
import org.apache.maven.artifact.Artifact;
import org.apache.maven.artifact.resolver.filter.ArtifactFilter;
import org.apache.maven.artifact.resolver.filter.ScopeArtifactFilter;
import org.apache.maven.model.Resource;
import org.apache.maven.plugin.MojoExecutionException;
Expand Down Expand Up @@ -133,7 +134,8 @@ public void execute() throws MojoExecutionException, MojoFailureException {
*/
private void buildLibraries(List<String> paths) throws IOException, MojoExecutionException {
Set<MavenArtifact> artifacts = getProjectArtfacts();

final ArtifactFilter classpathDependencyExcludesFilter = createClasspathDependencyExcludesFilter();
ScopeArtifactFilter filter = new ScopeArtifactFilter(Artifact.SCOPE_RUNTIME);
// List up IDs of Jenkins plugin dependencies
Set<String> jenkinsPlugins = new HashSet<>();
for (MavenArtifact artifact : artifacts) {
Expand Down Expand Up @@ -162,8 +164,10 @@ private void buildLibraries(List<String> paths) throws IOException, MojoExecutio
}
}

ScopeArtifactFilter filter = new ScopeArtifactFilter(Artifact.SCOPE_RUNTIME);
if (!artifact.isOptional() && filter.include(artifact.artifact)) {

if (!artifact.isOptional()
&& filter.include(artifact.artifact)
&& !classpathDependencyExcludesFilter.include(artifact.artifact)) {
paths.add(artifact.getFile().getPath());
}
}
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
<parent>
<groupId>com.qlangtech.tis</groupId>
<artifactId>tis-parent</artifactId>
<version>2.1.0</version>
<version>2.1.1</version>
</parent>

<groupId>com.qlangtech.tis</groupId>
Expand Down
2 changes: 1 addition & 1 deletion tis-builder-api/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
<dependency>
<groupId>com.qlangtech.tis</groupId>
<artifactId>tis-config</artifactId>
<version>1.0.1</version>
<version>1.0.2</version>
</dependency>

<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@
import com.qlangtech.tis.plugin.IdentityName;
import com.qlangtech.tis.plugin.KeyedPluginStore;
import com.qlangtech.tis.plugin.StoreResourceTypeGetter;
import com.qlangtech.tis.plugin.datax.transformer.OutputParameter;
import com.qlangtech.tis.plugin.datax.transformer.RecordTransformerRules;
import com.qlangtech.tis.plugin.datax.transformer.RecordTransformerRules.TransformerOverwriteCols;
import com.qlangtech.tis.plugin.ds.CMeta;
import com.qlangtech.tis.plugin.ds.DataSourceMeta;
import com.qlangtech.tis.plugin.ds.IColMetaGetter;
Expand Down Expand Up @@ -342,7 +344,8 @@ public static TabCols create(
IDBReservedKeys dbReservedKeys, TableMap tm, Optional<RecordTransformerRules> transformerRules) {

List<IColMetaGetter> cols = transformerRules.map((rule) -> {
return rule.overwriteCols(tm.getSourceCols()).getCols();
TransformerOverwriteCols<OutputParameter> outputParameters = rule.overwriteCols(tm.getSourceCols());
return outputParameters.getCols().stream().map((c) -> (IColMetaGetter) c).collect(Collectors.toList());
}).orElseGet(() -> {
return tm.getSourceCols().stream().map((c) -> c).collect(Collectors.toList());
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -403,9 +403,6 @@ private void generateDataXAndSQLDDLFile(File dataXCfgDir, IDataxReader reader, I
if (StringUtils.isEmpty(readerContext.getTaskName())) {
throw new IllegalStateException("readerContext.getTaskName() must be present");
}
// File configFile = new File(dataXCfgDir
// , readerContext.getReaderContextId() + File.separator + readerContext.getTaskName() +
// IDataxProcessor.DATAX_CREATE_DATAX_CFG_FILE_NAME_SUFFIX);
File configFile = DataXJobInfo.getJobPath(dataXCfgDir, readerContext.getReaderContextId(),
readerContext.getTaskName() + DataXCfgFile.DATAX_CREATE_DATAX_CFG_FILE_NAME_SUFFIX);
FileUtils.write(configFile, generateDataxConfig(readerContext, writer, reader, tableMapper
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public AbstractCreateTableSqlBuilder(

if (transformers.isPresent()) {
RecordTransformerRules transformerRules = transformers.get();
sourceCols = transformerRules.overwriteCols(sourceCols).getCols();
sourceCols = transformerRules.overwriteCols(sourceCols).getColsWithoutVirtualInfo();
}

this.cols = Collections.unmodifiableList(sourceCols.stream().map((c) -> createColWrapper(c)).collect(Collectors.toList()));
Expand Down Expand Up @@ -137,6 +137,11 @@ public String getSelectAllScript() {
return "SELECT " + cols.stream().map((c) -> builder.wrapWithEscape(c.getName()))
.collect(Collectors.joining(",")) + " FROM " + (builder.getCreateTableName().getEntityName());
}

public String getCountSelectScript(Optional<String> whereCriteria) {
return "SELECT count(1) FROM " + (builder.getCreateTableName().getEntityName())
+ whereCriteria.map((where) -> " WHERE " + where).orElse(StringUtils.EMPTY);
}
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ protected List<T> preProcessCols(List<String> pks, List<T> cols) {
return cols;
}

public static abstract class ColWrapper {
public static abstract class ColWrapper implements IColMetaGetter {
private final IColMetaGetter meta;
private final List<String> pks;

Expand All @@ -110,10 +110,12 @@ public ColWrapper(IColMetaGetter meta, List<String> pks) {
this.pks = Objects.requireNonNull(pks, "param pks can not be null");
}

@Override
public boolean isPk() {
return pks.contains(meta.getName());
}

@Override
public DataType getType() {
return this.meta.getType();
}
Expand All @@ -124,6 +126,7 @@ protected void appendExtraConstraint(BlockScriptBuffer ddlScript) {

}

@Override
public String getName() {
return this.meta.getName();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import com.qlangtech.tis.plugin.annotation.FormFieldType;
import com.qlangtech.tis.plugin.annotation.SubForm;
import com.qlangtech.tis.plugin.annotation.Validator;
import com.qlangtech.tis.plugin.datax.transformer.OutputParameter;
import com.qlangtech.tis.plugin.datax.transformer.RecordTransformerRules;
import com.qlangtech.tis.plugin.ds.CMeta;
import com.qlangtech.tis.plugin.ds.ColumnMetaData;
Expand Down Expand Up @@ -110,9 +111,11 @@ public List<IColMetaGetter> overwriteCols(IMessageHandler pluginCtx, boolean inc
List<IColMetaGetter> cols = null;
if (transformerRules.isPresent()) {
ITransformerBuildInfo transformerBuilder = transformerRules.get().createTransformerBuildInfo((IPluginContext) pluginCtx);
cols = transformerBuilder
.overwriteColsWithContextParams(this.getCols());
return includeContextParams ? cols : transformerBuilder.tranformerColsWithoutContextParams();

List<OutputParameter> outParams = includeContextParams
? transformerBuilder.overwriteColsWithContextParams(this.getCols())
: transformerBuilder.tranformerColsWithoutContextParams();
return outParams.stream().map((param) -> param).collect(Collectors.toList());

// OverwriteCols overwriteCols = transformerRules.get().overwriteCols(this.getCols());
// if (readerSource.isPresent()) {
Expand Down
Loading

0 comments on commit 97994ba

Please sign in to comment.