Add logic that checks that the response from service is a proper GetC…

…apabilities response

Remove recursion from loadCapabilitiesFromService

Let XMLStreamReader determine the charset in XML prolog

Remove encoding logic from GetGtWMSCapabilities by removing the prolog after decoding it succesfully

content-resources/src/main/java/flyway/oskari/V1_32_1__populate_capabilities_cache.java

@@ -35,7 +35,11 @@ public void migrate(Connection connection) throws SQLException {
             }
 
             keys.add(layerKey);
-            String data = CapabilitiesCacheService.loadCapabilitiesFromService(layer, null);
+            String data = CapabilitiesCacheService.loadCapabilitiesFromService(layer);
+            if (data == null) {
+                LOG.warn("Failed to read capabilities from service!");
+                continue;
+            }
             OskariLayerCapabilities draft = new OskariLayerCapabilities(
                     layer.getSimplifiedUrl(true),
                     layer.getType(),

service-map/src/main/java/fi/nls/oskari/service/capabilities/CapabilitiesCacheService.java

@@ -1,25 +1,32 @@
 package fi.nls.oskari.service.capabilities;
 
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.net.HttpURLConnection;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import javax.xml.stream.FactoryConfigurationError;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+
 import fi.nls.oskari.domain.map.OskariLayer;
 import fi.nls.oskari.log.LogFactory;
 import fi.nls.oskari.log.Logger;
 import fi.nls.oskari.service.OskariComponent;
 import fi.nls.oskari.service.ServiceException;
 import fi.nls.oskari.util.IOHelper;
 import fi.nls.oskari.util.PropertyUtil;
+import fi.nls.oskari.util.XmlHelper;
 
 public abstract class CapabilitiesCacheService extends OskariComponent {
 
     private static final Logger LOG = LogFactory.getLogger(CapabilitiesCacheService.class);
-    private static final String ENCODE_ATTRIBUTE =  "encoding=\"";
-
     private static final Map<String, String> TYPE_MAPPING = new HashMap<>(5);
     static {
         TYPE_MAPPING.put(OskariLayer.TYPE_WMS, "WMS");
@@ -37,6 +44,34 @@ public abstract class CapabilitiesCacheService extends OskariComponent {
     protected abstract void updateMultiple(final List<OskariLayerCapabilities> capabilities);
     protected abstract List<OskariLayerCapabilities> getAllOlderThan(final long maxAgeMs);
 
+    public void updateAllOlderThan(final long maxAgeMs) {
+        List<OskariLayerCapabilities> updates = new ArrayList<>();
+        for (OskariLayerCapabilities capabilities : getAllOlderThan(maxAgeMs)) {
+            String url = capabilities.getUrl();
+            String type = capabilities.getLayertype();
+            String version = capabilities.getVersion();
+            String dataOld = capabilities.getData();
+
+            OskariLayer layer = createTempOskariLayer(url, type, null, null, version);
+            String data = loadCapabilitiesFromService(layer);
+
+            if (data.isEmpty()) {
+                LOG.warn("Getting Capabilities from service failed for url:", url, "- skipping!");
+                continue;
+            }
+
+            if (dataOld.equals(data)) {
+                LOG.warn("New data is equal to old data for url:", url, "- skipping!");
+                continue;
+            }
+
+            capabilities.setData(data);
+            updates.add(capabilities);
+        }
+
+        updateMultiple(updates);
+    }
+
     public OskariLayerCapabilities getCapabilities(String url, String type, String version)
             throws ServiceException {
         return getCapabilities(url, type, null, null, version);
@@ -69,11 +104,6 @@ public OskariLayerCapabilities getCapabilities(final OskariLayer layer)
 
     public OskariLayerCapabilities getCapabilities(final OskariLayer layer, final boolean loadFromService)
             throws ServiceException {
-        return getCapabilities(layer, null, loadFromService);
-    }
-
-    public OskariLayerCapabilities getCapabilities(final OskariLayer layer, String encoding, final boolean loadFromService)
-            throws ServiceException {
         final String url = layer.getSimplifiedUrl(true);
         final String type = layer.getType();
         final String version = layer.getVersion();
@@ -87,124 +117,184 @@ public OskariLayerCapabilities getCapabilities(final OskariLayer layer, String e
         }
 
         // get xml from service
-        final String data = loadCapabilitiesFromService(layer, encoding, loadFromService);
+        final String data = loadCapabilitiesFromService(layer);
         if (data == null || data.trim().isEmpty()) {
             throw new ServiceException("Failed to load capabilities from service!");
         }
+
         try {
             return save(new OskariLayerCapabilities(url, type, version, data));
         } catch (IllegalArgumentException e) {
             throw new ServiceException("Failed to save capabilities: " + e.getMessage());
         }
     }
 
-    public static String loadCapabilitiesFromService(OskariLayer layer, String encoding) {
-        return loadCapabilitiesFromService(layer, encoding, false);
-    }
-
-    private static String loadCapabilitiesFromService(OskariLayer layer, String encoding, final boolean norecursion) {
+    public static String loadCapabilitiesFromService(OskariLayer layer) {
         final String url = contructCapabilitiesUrl(layer);
-        if (encoding == null) {
-            encoding = IOHelper.DEFAULT_CHARSET;
+        if (url.isEmpty()) {
+            return null;
         }
 
+        String encoding = null;
+        byte[] data = null;
         try {
             final HttpURLConnection conn = IOHelper.getConnection(url, layer.getUsername(), layer.getPassword());
             conn.setReadTimeout(TIMEOUT_MS);
 
             final int sc = conn.getResponseCode();
             if (sc != HttpURLConnection.HTTP_OK) {
-                LOG.warn("Unexpected Status code: ", sc, " url: ", url);
-                return "";
+                LOG.warn("Unexpected Status code:", sc, " url:", url);
+                return null;
             }
 
             final String contentType = conn.getContentType();
             if (contentType != null && contentType.toLowerCase().indexOf("xml") == -1) {
                 // not xml based on contentType
-                LOG.warn("Unexpected Content-Type: ", contentType, " url: ", url);
-                return "";
+                LOG.warn("Unexpected Content-Type:", contentType, "url:", url);
+                return null;
             }
 
-            final String response = IOHelper.readString(conn, encoding);
-            final String charset = getEncodingFromXml(response);
-
-            if (norecursion || charset == null || encoding.equalsIgnoreCase(charset)) {
-                return response;
-            }
-            return loadCapabilitiesFromService(layer, charset, true);
+            encoding = IOHelper.getCharset(conn);
+            data = IOHelper.readBytes(conn);
         } catch (IOException e) {
-            LOG.warn(e, "IOException occured, url: ", url, " error message: ", e.getMessage());
-            return "";
+            LOG.warn(e, "IOException occured, url:", url);
+            return null;
         }
-    }
 
-    public void updateAllOlderThan(final long maxAgeMs) {
-        List<OskariLayerCapabilities> updates = new ArrayList<>();
-        for (OskariLayerCapabilities capabilities : getAllOlderThan(maxAgeMs)) {
-            String url = capabilities.getUrl();
-            String type = capabilities.getLayertype();
-            String version = capabilities.getVersion();
-            String dataOld = capabilities.getData();
-
-            OskariLayer layer = createTempOskariLayer(url, type, null, null, version);
-            String data = loadCapabilitiesFromService(layer, IOHelper.DEFAULT_CHARSET);
-
-            if (data.isEmpty()) {
-                LOG.warn("Getting Capabilities from service failed for url:", url, "- skipping!");
-                continue;
+        try {
+            XMLInputFactory xif = XMLInputFactory.newFactory();
+            xif.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, true);
+            XMLStreamReader xsr = xif.createXMLStreamReader(new ByteArrayInputStream(data));
+
+            // Check XML prolog for character encoding
+            String xmlEncoding = xsr.getCharacterEncodingScheme();
+            if (xmlEncoding != null) {
+                if (encoding != null && !xmlEncoding.equalsIgnoreCase(encoding)) {
+                    LOG.error("Content-Type header specified a different encoding than XML prolog!");
+                    return null;
+                }
+                encoding = xmlEncoding;
+            }
+            if (encoding == null) {
+                LOG.debug("Charset wasn't set on either the Content-Type or the XML prolog"
+                        + "using UTF-8 as default value");
+                encoding = IOHelper.DEFAULT_CHARSET;
             }
 
-            if (dataOld.equals(data)) {
-                LOG.warn("New data is equal to old data for url:", url, "- skipping!");
-                continue;
+            // Check that the response is what we expect
+            if (!checkCapabilities(xsr, layer.getType(), layer.getVersion())) {
+                return null;
             }
 
-            capabilities.setData(data);
-            updates.add(capabilities);
+            // Convert "utf-8" to "UTF-8" for example
+            encoding = encoding.toUpperCase();
+            String xml = new String(data, encoding);
+            // Strip the potential prolog from XML so that we
+            // don't have to worry about the specified charset
+            return XmlHelper.stripPrologFromXML(xml);
+        } catch (FactoryConfigurationError | XMLStreamException e) {
+            LOG.warn(e, "Failed to parse XML from response");
+        } catch (UnsupportedEncodingException e) {
+            LOG.warn(e, "Failed to Encode byte[] to String encoding:", encoding);
         }
-
-        updateMultiple(updates);
+        return null;
     }
 
     public static String contructCapabilitiesUrl(final OskariLayer layer) {
         if (layer == null) {
             return "";
         }
-        final String url  = layer.getSimplifiedUrl(true);
+
+        final String url = layer.getSimplifiedUrl(true);
+        final String urlLC = url.toLowerCase();
+        final String serviceType = TYPE_MAPPING.get(layer.getType());
+
         final Map<String, String> params = new HashMap<String, String>();
         // check existing params
-        if(!url.toLowerCase().contains("service=")) {
-            params.put("service", TYPE_MAPPING.get(layer.getType()));
+        if (!urlLC.contains("service=")) {
+            params.put("service", serviceType);
         }
-        if(!url.toLowerCase().contains("request=")) {
+        if (!urlLC.contains("request=")) {
             params.put("request", "GetCapabilities");
         }
-        if(!url.toLowerCase().contains("version=") && layer.getVersion() != null) {
-            params.put("version", layer.getVersion());
+        if (!urlLC.contains("version=") && layer.getVersion() != null) {
+            params.put(getVersionNegotiationKey(serviceType), layer.getVersion());
         }
 
         return IOHelper.constructUrl(url, params);
     }
 
-    // TODO: maybe use some lib instead?
-    public static String getEncodingFromXml(final String response) {
-        if(response == null) {
-            return null;
+    private static String getVersionNegotiationKey(String service) {
+        if (service != null) {
+            switch (service) {
+            case "WMS":
+                return "version";
+            case "WFS":
+            case "WMTS":
+                return "acceptVersions";
+            }
         }
+        return "";
+    }
 
-        final String[] processingSplit  = response.split("\\?>");
+    private static boolean checkCapabilities(XMLStreamReader xsr, String type, String version) {
+        if (!advanceToRootElement(xsr)) {
+            // Could not advance to root element
+            return false;
+        }
+        String ns = xsr.getNamespaceURI();
+        String name = xsr.getLocalName();
+        return checkCapabilities(type, version, ns, name);
+    }
 
-        if (processingSplit == null || processingSplit.length == 0) {
-            return null;
+    private static boolean advanceToRootElement(XMLStreamReader xsr) {
+        try {
+            if (xsr.nextTag() != XMLStreamConstants.START_DOCUMENT) {
+                LOG.warn("Document did not start with a START_DOCUMENT!");
+                return false;
+            }
+            if (xsr.nextTag() != XMLStreamConstants.START_ELEMENT) {
+                LOG.warn("Could not find root element!");
+                return false;
+            }
+            return true;
+        } catch (XMLStreamException e) {
+            LOG.warn(e, "Failed to find root element!");
+            return false;
         }
+    }
 
-        int encodeAttributeStart = processingSplit[0].indexOf(ENCODE_ATTRIBUTE);
-        if (encodeAttributeStart > 0) {
-            encodeAttributeStart = encodeAttributeStart + ENCODE_ATTRIBUTE.length();
-            return processingSplit[0].substring(encodeAttributeStart, processingSplit[0].indexOf('"', encodeAttributeStart));
+    private static boolean checkCapabilities(String type, String version, String ns, String name) {
+        LOG.debug("Checking capabilities, type:", type, "version:", version, 
+                "namespace", ns, "root element", name);
+
+        switch (type) {
+        case OskariLayer.TYPE_WMS:
+            if (version == null) {
+                // Layer didn't specify a version - response could could be anything
+                return isWMS130Capabilities(ns, name) || isWMSLessThan130Capabilities(ns, name);
+            } else if ("1.3.0".equals(version)) {
+                return isWMS130Capabilities(ns, name);
+            } else {
+                return isWMSLessThan130Capabilities(ns, name);
+            }
+        case OskariLayer.TYPE_WFS:
+            return ns.startsWith("http://www.opengis.net/wfs/") && "WFS_Capabilities".equals(name);
+        case OskariLayer.TYPE_WMTS:
+            return ns.startsWith("http://www.opengis.net/wmts/") && "Capabilities".equals(name);
         }
+        return false;
+    }
 
-        return null;
+    private static boolean isWMSLessThan130Capabilities(String ns, String name) {
+        return ns == null
+                && "WMT_MS_Capabilities".equals(name);
+    }
+
+    private static boolean isWMS130Capabilities(String ns, String name) {
+        return ns != null 
+                && ns.startsWith("http://www.opengis.net/wms/") 
+                && "WMS_Capabilities".equals(name);
     }
 
 }

service-map/src/main/java/fi/nls/oskari/wms/GetGtWMSCapabilities.java

@@ -46,17 +46,14 @@ private GetGtWMSCapabilities() {
     }
 
     // based on https://github.com/geotools/geotools/blob/master/modules/extension/wms/src/test/java/org/geotools/data/wms/test/WMS1_0_0_OnlineTest.java#L253-L276
-    public static WMSCapabilities createCapabilities(String xml, String encoding) {
+    public static WMSCapabilities createCapabilities(String xml) {
         if(xml == null || xml.isEmpty()) {
             return null;
         }
-        if (encoding == null) {
-            encoding = StandardCharsets.UTF_8.toString();
-        }
         final Map hints = new HashMap();
         hints.put(DocumentHandler.DEFAULT_NAMESPACE_HINT_KEY, WMSSchema.getInstance());
         hints.put(DocumentFactory.VALIDATION_HINT, false);
-        try(InputStream stream = new ByteArrayInputStream(xml.getBytes(encoding))) {
+        try(InputStream stream = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) {
             final Object object = DocumentFactory.getInstance(stream, hints, Level.WARNING);
             if(object instanceof WMSCapabilities) {
                 return (WMSCapabilities) object;
@@ -85,11 +82,10 @@ public static JSONObject getWMSCapabilities(final String rurl, final String user
             String capabilitiesXML = capabilities.getData();
             if(capabilitiesXML == null || capabilitiesXML.trim().isEmpty()) {
                 // retry from service - might get empty xml from db
-                capabilities = service.getCapabilities(rurl, "wmslayer", user, pwd, version, true);
+                capabilities = service.getCapabilities(rurl, OskariLayer.TYPE_WMS, user, pwd, version, true);
                 capabilitiesXML = capabilities.getData();
             }
-            String encoding = CapabilitiesCacheService.getEncodingFromXml(capabilitiesXML);
-            WMSCapabilities caps = createCapabilities(capabilitiesXML, encoding);
+            WMSCapabilities caps = createCapabilities(capabilitiesXML);
             // caps to json
             return parseLayer(caps.getLayer(), rurl, caps, capabilitiesXML, currentCrs, false);
         } catch (Exception ex) {