Skip to content

Commit 87bd4dd

Browse files
mike-podolskiy90mike-podolskiy90
committed
Merge remote-tracking branch 'frictionless/main'
# Conflicts: # pom.xml
2 parents 2281b55 + e2a41c3 commit 87bd4dd

File tree

13 files changed

+984
-74
lines changed

13 files changed

+984
-74
lines changed

‎pom.xml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<modelVersion>4.0.0</modelVersion>
44
<groupId>io.frictionlessdata</groupId>
55
<artifactId>datapackage-java</artifactId>
6-
<version>0.9.2.2-gbif-SNAPSHOT</version>
6+
<version>0.9.7-gbif-SNAPSHOT</version>
77
<packaging>jar</packaging>
88

99
<issueManagement>
@@ -36,8 +36,8 @@
3636
<maven.compiler.source>${java.version}</maven.compiler.source>
3737
<maven.compiler.target>${java.version}</maven.compiler.target>
3838
<maven.compiler.compiler>${java.version}</maven.compiler.compiler>
39-
<tableschema-java-version>0.9.2.1-gbif</tableschema-java-version>
40-
<junit.version>5.12.2</junit.version>
39+
<tableschema-java-version>0.9.6-gbif-SNAPSHOT</tableschema-java-version>
40+
<junit.version>5.13.2</junit.version>
4141
<slf4j-simple.version>2.0.17</slf4j-simple.version>
4242
<apache-commons-collections4.version>4.5.0</apache-commons-collections4.version>
4343
<maven-compiler-plugin.version>3.14.0</maven-compiler-plugin.version>
@@ -51,9 +51,9 @@
5151
<maven-release-plugin.version>3.1.1</maven-release-plugin.version>
5252
<nexus-staging-maven-plugin.version>1.7.0</nexus-staging-maven-plugin.version>
5353
<coveralls-maven-plugin.version>4.3.0</coveralls-maven-plugin.version>
54-
<dependency-check-maven.version>12.1.1</dependency-check-maven.version>
54+
<dependency-check-maven.version>12.1.3</dependency-check-maven.version>
5555
<jacoco-maven-plugin.version>0.8.13</jacoco-maven-plugin.version>
56-
<spotbugs-maven-plugin.version>4.9.3.0</spotbugs-maven-plugin.version>
56+
<spotbugs-maven-plugin.version>4.9.3.2</spotbugs-maven-plugin.version>
5757
</properties>
5858

5959
<repositories>

‎src/main/java/io/frictionlessdata/datapackage/Package.java‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
import io.frictionlessdata.datapackage.resource.Resource;
1818
import io.frictionlessdata.tableschema.exception.JsonParsingException;
1919
import io.frictionlessdata.tableschema.exception.ValidationException;
20-
import io.frictionlessdata.tableschema.io.LocalFileReference;
2120
import io.frictionlessdata.tableschema.util.JsonUtil;
2221
import org.apache.commons.collections.list.UnmodifiableList;
2322
import org.apache.commons.collections.set.UnmodifiableSet;
@@ -609,16 +608,6 @@ public void write (File outputDir, Consumer<Path> callback, boolean zipCompresse
609608
r.writeData(outFs.getPath(parentDirName ));
610609
r.writeSchema(outFs.getPath(parentDirName));
611610
r.writeDialect(outFs.getPath(parentDirName));
612-
613-
// write out dialect file only if not null or URL
614-
615-
/*Dialect dia = r.getDialect();
616-
if (null != dia) {
617-
String dialectP = r.getPathForWritingDialect();
618-
Path dialectPath = outFs.getPath(parentDirName + File.separator + dialectP);
619-
dia.writeDialect(dialectPath);
620-
dia.setReference(new LocalFileReference(new File(dialectP)));
621-
}*/
622611
}
623612
writeDescriptor(outFs, parentDirName);
624613

@@ -787,7 +776,7 @@ private void setJson(ObjectNode jsonNodeSource) throws Exception {
787776
ObjectNode resourceJson = (ObjectNode) resourcesJsonArray.get(i);
788777
Resource resource = null;
789778
try {
790-
resource = Resource.build(resourceJson, basePath, isArchivePackage);
779+
resource = Resource.fromJSON(resourceJson, basePath, isArchivePackage);
791780
} catch (DataPackageException dpe) {
792781
if(this.strictValidation){
793782
this.jsonObject = null;
@@ -1017,6 +1006,9 @@ private static URL getParentUrl(URL urlSource) throws URISyntaxException, Malfor
10171006

10181007
// https://stackoverflow.com/a/47595502/2535335
10191008
private static boolean isArchive(File f) throws IOException {
1009+
if ((null == f) || (!f.exists()) || (!f.isFile())) {
1010+
return false;
1011+
}
10201012
int fileSignature;
10211013
RandomAccessFile raf = new RandomAccessFile(f, "r");
10221014
fileSignature = raf.readInt();

‎src/main/java/io/frictionlessdata/datapackage/resource/AbstractDataResource.java‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@
33
import com.fasterxml.jackson.annotation.JsonIgnore;
44
import com.fasterxml.jackson.annotation.JsonInclude;
55
import com.fasterxml.jackson.annotation.JsonProperty;
6-
import io.frictionlessdata.datapackage.Dialect;
6+
import io.frictionlessdata.datapackage.Package;
77
import io.frictionlessdata.datapackage.exceptions.DataPackageException;
8+
import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException;
89
import io.frictionlessdata.tableschema.Table;
910

1011
import java.io.IOException;
11-
import java.nio.file.Path;
1212
import java.util.ArrayList;
1313
import java.util.HashSet;
1414
import java.util.List;
@@ -77,6 +77,28 @@ public Set<String> getDatafileNamesForWriting() {
7777
return names;
7878
}
7979

80+
@Override
81+
public void validate(Package pkg) throws DataPackageValidationException {
82+
super.validate(pkg);
83+
try {
84+
if (getRawData() == null) {
85+
throw new DataPackageValidationException("Data resource must have data");
86+
}
87+
88+
if (getFormat() == null) {
89+
throw new DataPackageValidationException("Data resource must specify a format");
90+
}
91+
} catch (Exception ex) {
92+
if (ex instanceof DataPackageValidationException) {
93+
errors.add((DataPackageValidationException) ex);
94+
}
95+
else {
96+
errors.add(new DataPackageValidationException(ex));
97+
}
98+
}
99+
}
100+
101+
80102
@JsonIgnore
81103
abstract String getResourceFormat();
82104
}

‎src/main/java/io/frictionlessdata/datapackage/resource/AbstractReferencebasedResource.java‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,18 @@
44
import com.fasterxml.jackson.annotation.JsonInclude;
55
import com.fasterxml.jackson.annotation.JsonProperty;
66
import com.fasterxml.jackson.databind.JsonNode;
7+
import io.frictionlessdata.datapackage.Package;
8+
import io.frictionlessdata.datapackage.exceptions.DataPackageException;
9+
import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException;
710
import io.frictionlessdata.tableschema.Table;
811
import io.frictionlessdata.tableschema.tabledatasource.TableDataSource;
912
import io.frictionlessdata.tableschema.util.JsonUtil;
1013

1114
import java.io.ByteArrayOutputStream;
15+
import java.io.File;
1216
import java.io.IOException;
1317
import java.io.InputStream;
18+
import java.net.URL;
1419
import java.util.*;
1520
import java.util.stream.Collectors;
1621

@@ -90,6 +95,68 @@ public Set<String> getDatafileNamesForWriting() {
9095
}).collect(Collectors.toSet());
9196
}
9297

98+
@Override
99+
public void validate(Package pkg) throws DataPackageValidationException {
100+
super.validate(pkg);
101+
List<T> paths = new ArrayList<>(getPaths());
102+
try {
103+
if (paths.isEmpty()) {
104+
throw new DataPackageValidationException("File- or URL-based resource must have at least one path");
105+
}
106+
107+
for (T path : paths) {
108+
String name = null;
109+
if (path instanceof File) {
110+
name = ((File) path).getName();
111+
} else if (path instanceof URL) {
112+
name = ((URL) path).getPath();
113+
}
114+
if (name == null || name.trim().isEmpty()) {
115+
throw new DataPackageValidationException("Resource path cannot be null or empty");
116+
}
117+
}
118+
} catch (Exception ex) {
119+
if (ex instanceof DataPackageValidationException) {
120+
errors.add((DataPackageValidationException) ex);
121+
}
122+
else {
123+
errors.add(new DataPackageValidationException(ex));
124+
}
125+
}
126+
}
127+
128+
129+
static String sniffFormat(Collection<?> paths) {
130+
Set<String> foundFormats = new HashSet<>();
131+
for (Object p : paths) {
132+
String name;
133+
if (p instanceof String) {
134+
name = (String) p;
135+
} else if (p instanceof File) {
136+
name = ((File) p).getName();
137+
} else if (p instanceof URL) {
138+
name = ((URL) p).getPath();
139+
} else {
140+
throw new DataPackageException("Unsupported path type: " + p.getClass().getName());
141+
}
142+
if (name.toLowerCase().endsWith(TableDataSource.Format.FORMAT_CSV.getLabel())) {
143+
foundFormats.add(TableDataSource.Format.FORMAT_CSV.getLabel());
144+
} else if (name.toLowerCase().endsWith(TableDataSource.Format.FORMAT_JSON.getLabel())) {
145+
foundFormats.add(TableDataSource.Format.FORMAT_JSON.getLabel());
146+
} else {
147+
// something else -> not a tabular resource
148+
int pos = name.lastIndexOf('.');
149+
return name.substring(pos + 1).toLowerCase();
150+
}
151+
}
152+
if (foundFormats.size() > 1) {
153+
throw new DataPackageException("Resources cannot be mixed JSON/CSV");
154+
}
155+
if (foundFormats.isEmpty())
156+
return TableDataSource.Format.FORMAT_CSV.getLabel();
157+
return foundFormats.iterator().next();
158+
}
159+
93160
abstract Table createTable(T reference) throws Exception;
94161

95162
abstract String getStringRepresentation(T reference);

‎src/main/java/io/frictionlessdata/datapackage/resource/AbstractResource.java‎

Lines changed: 53 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,6 @@
66
import com.fasterxml.jackson.annotation.JsonProperty;
77
import com.fasterxml.jackson.core.JsonProcessingException;
88
import com.fasterxml.jackson.databind.ObjectMapper;
9-
import com.fasterxml.jackson.databind.node.ArrayNode;
10-
import com.fasterxml.jackson.databind.node.ObjectNode;
119
import io.frictionlessdata.datapackage.Dialect;
1210
import io.frictionlessdata.datapackage.JSONBase;
1311
import io.frictionlessdata.datapackage.Package;
@@ -34,12 +32,11 @@
3432
import org.apache.commons.csv.CSVFormat;
3533
import org.apache.commons.csv.CSVPrinter;
3634

37-
import java.io.FileOutputStream;
38-
import java.io.IOException;
39-
import java.io.Writer;
35+
import java.io.*;
4036
import java.net.URI;
4137
import java.net.URISyntaxException;
4238
import java.nio.charset.StandardCharsets;
39+
import java.nio.file.FileSystem;
4340
import java.nio.file.Files;
4441
import java.nio.file.Path;
4542
import java.util.*;
@@ -449,9 +446,33 @@ public void checkRelations(Package pkg) {
449446
}
450447

451448
public void validate(Package pkg) {
452-
if (null == tables)
453-
return;
449+
454450
try {
451+
// Validate required fields
452+
if (getName() == null || getName().trim().isEmpty()) {
453+
throw new DataPackageValidationException("Resource must have a name");
454+
}
455+
456+
// Validate name format (alphanumeric, dash, underscore only)
457+
if (!getName().matches("^[a-zA-Z0-9_-]+$")) {
458+
throw new DataPackageValidationException("Resource name must contain only alphanumeric characters, dashes, and underscores");
459+
}
460+
461+
// Validate profile
462+
String profile = getProfile();
463+
if (profile != null && !isValidProfile(profile)) {
464+
throw new DataPackageValidationException("Invalid resource profile: " + profile);
465+
}
466+
467+
if (null != schema) {
468+
try {
469+
schema.validate();
470+
} catch (DataPackageValidationException e) {
471+
throw new DataPackageValidationException("Schema validation failed for resource " + getName() + ": " + e.getMessage(), e);
472+
}
473+
}
474+
if (null == tables)
475+
return;
455476
// will validate schema against data
456477
tables.forEach(Table::validate);
457478
checkRelations(pkg);
@@ -728,7 +749,7 @@ public void setShouldSerializeFullSchema(boolean serializeFullSchema) {
728749

729750
@Override
730751
public void setSerializationFormat(String format) {
731-
if ((format.equals(TableDataSource.Format.FORMAT_JSON.getLabel()))
752+
if ((null == format) || (format.equals(TableDataSource.Format.FORMAT_JSON.getLabel()))
732753
|| format.equals(TableDataSource.Format.FORMAT_CSV.getLabel())) {
733754
this.serializationFormat = format;
734755
} else
@@ -751,7 +772,7 @@ public String getSerializationFormat() {
751772

752773
public abstract Set<String> getDatafileNamesForWriting();
753774

754-
private List<Table> ensureDataLoaded () throws Exception {
775+
List<Table> ensureDataLoaded () throws Exception {
755776
if (null == tables) {
756777
tables = readData();
757778
}
@@ -774,16 +795,19 @@ public void writeData(Writer out) throws Exception {
774795
@Override
775796
public void writeData(Path outputDir) throws Exception {
776797
Dialect lDialect = (null != dialect) ? dialect : Dialect.DEFAULT;
777-
List<Table> tables = getTables();
798+
boolean isNonTabular = ((profile != null) && (profile.equals(Profile.PROFILE_DATA_RESOURCE_DEFAULT)));
799+
isNonTabular = (isNonTabular | (null == serializationFormat));
800+
List<Table> tables = isNonTabular ? null : getTables();
801+
778802
Set<String> paths = getDatafileNamesForWriting();
779803

780804
int cnt = 0;
781805
for (String fName : paths) {
782806
String fileName = fName+"."+getSerializationFormat();
783-
Table t = tables.get(cnt++);
784807
Path p;
808+
FileSystem fileSystem = outputDir.getFileSystem();
785809
if (outputDir.toString().isEmpty()) {
786-
p = outputDir.getFileSystem().getPath(fileName);
810+
p = fileSystem.getPath(fileName);
787811
} else {
788812
p = outputDir.resolve(fileName);
789813
}
@@ -792,25 +816,34 @@ public void writeData(Path outputDir) throws Exception {
792816
}
793817
Files.deleteIfExists(p);
794818

795-
// if the serializationFormat is set, serialize the data to JSON/CSV file
796-
if (null != serializationFormat) {
819+
820+
if (isNonTabular) {
821+
byte [] data = (byte[])this.getRawData();
822+
try (OutputStream out = Files.newOutputStream(p)) {
823+
out.write(data);
824+
}
825+
} else {
826+
Table t = tables.get(cnt++);
797827
try (Writer wr = Files.newBufferedWriter(p, StandardCharsets.UTF_8)) {
798828
if (serializationFormat.equals(TableDataSource.Format.FORMAT_CSV.getLabel())) {
799829
t.writeCsv(wr, lDialect.toCsvFormat());
800830
} else if (serializationFormat.equals(TableDataSource.Format.FORMAT_JSON.getLabel())) {
801831
wr.write(t.asJson());
802832
}
803833
}
804-
} else {
805-
// if serializationFormat is not set (probably non-tabular data), serialize the data to a binary file
806-
byte [] data = (byte[])this.getRawData();
807-
try (FileOutputStream fos = new FileOutputStream(p.toFile())){
808-
fos.write(data);
809-
}
810834
}
811835
}
812836
}
813837

838+
839+
private static boolean isValidProfile(String profile) {
840+
return profile.equals(Profile.PROFILE_DATA_RESOURCE_DEFAULT) ||
841+
profile.equals(Profile.PROFILE_TABULAR_DATA_RESOURCE) ||
842+
profile.startsWith("http://") ||
843+
profile.startsWith("https://");
844+
}
845+
846+
814847
/**
815848
* Write the Table as CSV into a file inside `outputDir`.
816849
*

0 commit comments

Comments
 (0)