diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 372c3988c8..c48fd6055f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,3 +1,3 @@ ## Changelog for v2.2.2 - -- Declare the schema for the spatial_ref_sys table +- Read CPG file attached to the shapefile- +- Declare the schema for the spatial_ref_sys table \ No newline at end of file diff --git a/h2gis-functions/src/main/java/org/h2gis/functions/io/dbf/DBFDriverFunction.java b/h2gis-functions/src/main/java/org/h2gis/functions/io/dbf/DBFDriverFunction.java index 8200aa06e1..233a77c898 100644 --- a/h2gis-functions/src/main/java/org/h2gis/functions/io/dbf/DBFDriverFunction.java +++ b/h2gis-functions/src/main/java/org/h2gis/functions/io/dbf/DBFDriverFunction.java @@ -229,6 +229,8 @@ public String[] importFile(Connection connection, String tableReference, File stmt.execute("DROP TABLE IF EXISTS " + outputTable); stmt.close(); } + //Check if a cpg file exists + //TODO: DBFDriver dbfDriver = new DBFDriver(); dbfDriver.initDriverFromFile(fileName, options); DbaseFileHeader dbfHeader = dbfDriver.getDbaseFileHeader(); diff --git a/h2gis-functions/src/main/java/org/h2gis/functions/io/dbf/internal/DbaseFileHeader.java b/h2gis-functions/src/main/java/org/h2gis/functions/io/dbf/internal/DbaseFileHeader.java index eed31274f8..08565c3a08 100644 --- a/h2gis-functions/src/main/java/org/h2gis/functions/io/dbf/internal/DbaseFileHeader.java +++ b/h2gis-functions/src/main/java/org/h2gis/functions/io/dbf/internal/DbaseFileHeader.java @@ -503,15 +503,49 @@ public void readHeader(FileChannel channel,String forceEncoding) throws IOExcept // skip / skip thesreserved bytes in the header. in.skip(17); - // read Language driver - byte lngDriver = in.get(); - String encoding = CODE_PAGE_ENCODING.get(lngDriver); - if(encoding!=null && forceEncoding == null) { - this.fileEncoding = encoding; - } - // skip reserved - in.skip(2); + // read Language driver + byte lngDriver = in.get(); + String encoding = CODE_PAGE_ENCODING.get(lngDriver); + // skip reserved + in.skip(2); + //FROM : https://github.com/OSGeo/gdal/blob/master/ogr/ogrsf_frmts/shape/ogrshapelayer.cpp#L526 + if(forceEncoding!=null){ + forceEncoding = forceEncoding.toUpperCase(); + try { + //Only code page + int codepage = Integer.valueOf(forceEncoding); + if ((codepage >= 437 && codepage <= 950) || (codepage >= 1250 && codepage <= 1258)) { + this.fileEncoding = new StringBuffer("CP").append(forceEncoding).toString(); + } + else if (forceEncoding.startsWith("8859")) + { + if(forceEncoding.startsWith("-", 4)){ + this.fileEncoding ="ISO-8859-5"; + }else{ + this.fileEncoding ="ISO-8859-4"; + } + } + else if (forceEncoding.startsWith("UTF-8")||forceEncoding.startsWith("UTF8")) + { + this.fileEncoding ="UTF-8"; + } + else if (forceEncoding.startsWith( "ANSI 1251")) { + this.fileEncoding = "CP1251"; + } + else + { + // Try just using the CPG value directly. Works for stuff like Big5. + fileEncoding = forceEncoding; + } + }catch (NumberFormatException ex){ + //Nothing to do + } + }else { + if(encoding!=null){ + this.fileEncoding = encoding; + } + } // calculate the number of Fields in the header fieldCnt = (headerLength - FILE_DESCRIPTOR_SIZE - 1) / FILE_DESCRIPTOR_SIZE; diff --git a/h2gis-functions/src/main/java/org/h2gis/functions/io/shp/internal/SHPDriver.java b/h2gis-functions/src/main/java/org/h2gis/functions/io/shp/internal/SHPDriver.java index cafc44d0a7..938866bd34 100644 --- a/h2gis-functions/src/main/java/org/h2gis/functions/io/shp/internal/SHPDriver.java +++ b/h2gis-functions/src/main/java/org/h2gis/functions/io/shp/internal/SHPDriver.java @@ -32,6 +32,7 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; /** * Merge ShapeFileReader and DBFReader. @@ -57,6 +58,7 @@ public class SHPDriver implements FileDriver { private int geometryFieldIndex = 0; private ShapeType shapeType; public File prjFile; + public File cpgFile; private int srid =0; @@ -168,6 +170,9 @@ else if(path.equals(fileNamePrefix+".dbf")){ else if(path.equals(fileNamePrefix+".prj")){ prjFile = entry.toFile(); return true; + } else if(path.equals(fileNamePrefix+".cpg")){ + cpgFile = entry.toFile(); + return true; } return false; } @@ -180,6 +185,14 @@ else if(path.equals(fileNamePrefix+".prj")){ } } if(dbfFile != null) { + //Read the CPG file if exists + if(cpgFile!=null){ + BufferedReader br = Files.newBufferedReader(cpgFile.toPath()); + String codePage; + if ((codePage = br.readLine()) != null && forceEncoding==null) { + forceEncoding = codePage.trim(); + } + } dbfDriver.initDriverFromFile(dbfFile, forceEncoding); } else { throw new IllegalArgumentException("DBF File not found"); diff --git a/h2gis-functions/src/test/java/org/h2gis/functions/io/shp/SHPEngineTest.java b/h2gis-functions/src/test/java/org/h2gis/functions/io/shp/SHPEngineTest.java index de3c7e67d4..fe8fe519ee 100644 --- a/h2gis-functions/src/test/java/org/h2gis/functions/io/shp/SHPEngineTest.java +++ b/h2gis-functions/src/test/java/org/h2gis/functions/io/shp/SHPEngineTest.java @@ -449,4 +449,32 @@ public void linkedShpSpatialIndexFlatQueryTest() throws SQLException { assertTrue(rs.getString(1).contains("PK_INDEX"), "Expected contains PK_INDEX but result is " + rs.getString(1)); } } + + @Test + public void readSHPWithCPGFileTest() throws SQLException { + Statement st = connection.createStatement(); + st.execute("drop table if exists shptable"); + st.execute("CALL FILE_TABLE('"+SHPEngineTest.class.getResource("urock_buildings.shp").getPath()+"', 'SHPtable');"); + try ( // Query declared Table columns + ResultSet rs = st.executeQuery("SELECT * FROM SHPTABLE where objektiden= '71193131-5b61-4d65-a661-fefb173bfd86';")) { + assertTrue(rs.next()); + assertEquals("71193131-5b61-4d65-a661-fefb173bfd86", rs.getString("objektiden")); + assertEquals("Bostad;Småhus friliggande",rs.getString("andamal1")); + } + st.execute("drop table shptable"); + } + + @Test + public void readSHPWithCPGFordeTest() throws SQLException { + Statement st = connection.createStatement(); + st.execute("drop table if exists shptable"); + st.execute("CALL SHPREAD( '"+SHPEngineTest.class.getResource("urock_buildings.shp").getPath()+"', 'SHPtable','windows-1252');"); + try ( // Query declared Table columns + ResultSet rs = st.executeQuery("SELECT * FROM SHPTABLE where objektiden= '3110e3c4-638c-485e-b5fe-a827ebafd071';")) { + assertTrue(rs.next()); + assertEquals("3110e3c4-638c-485e-b5fe-a827ebafd071", rs.getString("objektiden")); + assertEquals("Industri;Tillverkning",rs.getString("andamal1")); + } + st.execute("drop table shptable"); + } } diff --git a/h2gis-functions/src/test/resources/org/h2gis/functions/io/shp/urock_buildings.cpg b/h2gis-functions/src/test/resources/org/h2gis/functions/io/shp/urock_buildings.cpg new file mode 100644 index 0000000000..03ce6dfae7 --- /dev/null +++ b/h2gis-functions/src/test/resources/org/h2gis/functions/io/shp/urock_buildings.cpg @@ -0,0 +1 @@ +1252 \ No newline at end of file diff --git a/h2gis-functions/src/test/resources/org/h2gis/functions/io/shp/urock_buildings.dbf b/h2gis-functions/src/test/resources/org/h2gis/functions/io/shp/urock_buildings.dbf new file mode 100644 index 0000000000..4439ae5561 Binary files /dev/null and b/h2gis-functions/src/test/resources/org/h2gis/functions/io/shp/urock_buildings.dbf differ diff --git a/h2gis-functions/src/test/resources/org/h2gis/functions/io/shp/urock_buildings.prj b/h2gis-functions/src/test/resources/org/h2gis/functions/io/shp/urock_buildings.prj new file mode 100644 index 0000000000..2ab2e0977a --- /dev/null +++ b/h2gis-functions/src/test/resources/org/h2gis/functions/io/shp/urock_buildings.prj @@ -0,0 +1 @@ +PROJCS["SWEREF99_12_00",GEOGCS["GCS_SWEREF99",DATUM["D_SWEREF99",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",150000.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",12.0],PARAMETER["Scale_Factor",1.0],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]] \ No newline at end of file diff --git a/h2gis-functions/src/test/resources/org/h2gis/functions/io/shp/urock_buildings.shp b/h2gis-functions/src/test/resources/org/h2gis/functions/io/shp/urock_buildings.shp new file mode 100644 index 0000000000..ab30efe202 Binary files /dev/null and b/h2gis-functions/src/test/resources/org/h2gis/functions/io/shp/urock_buildings.shp differ diff --git a/h2gis-functions/src/test/resources/org/h2gis/functions/io/shp/urock_buildings.shx b/h2gis-functions/src/test/resources/org/h2gis/functions/io/shp/urock_buildings.shx new file mode 100644 index 0000000000..58dad8d0e9 Binary files /dev/null and b/h2gis-functions/src/test/resources/org/h2gis/functions/io/shp/urock_buildings.shx differ