Skip to content

Commit

Permalink
Merge pull request #86 from jampukka/feature/timeseries-capabilities
Browse files Browse the repository at this point in the history 
Feature/update-capabilities-automatically
  • Loading branch information
@ZakarFin
ZakarFin authored Nov 14, 2017
2 parents 61d2a84 + db0cf78 commit 104702c
Show file tree
Hide file tree
Showing 34 changed files with 1,553 additions and 1,011 deletions.
69 changes: 32 additions & 37 deletions content-resources/src/main/java/flyway/oskari/V1_32_1__populate_capabilities_cache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,9 @@
import fi.nls.oskari.domain.map.OskariLayer;
import fi.nls.oskari.log.LogFactory;
import fi.nls.oskari.log.Logger;
import fi.nls.oskari.service.OskariComponentManager;
import fi.nls.oskari.service.capabilities.CapabilitiesCacheService;
import fi.nls.oskari.service.capabilities.OskariLayerCapabilities;
import fi.nls.oskari.service.ServiceException;
import org.flywaydb.core.api.migration.jdbc.JdbcMigration;

import java.io.IOException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
Expand All @@ -18,44 +15,42 @@
import java.util.List;
import java.util.Set;

/**
* Created by SMAKINEN on 25.8.2015.
*/
public class V1_32_1__populate_capabilities_cache implements JdbcMigration {

private static final Logger LOG = LogFactory.getLogger(V1_32_1__populate_capabilities_cache.class);

private CapabilitiesCacheService capabilitiesService;

public void migrate(Connection connection) throws SQLException {
capabilitiesService = OskariComponentManager.getComponentOfType(CapabilitiesCacheService.class);
List<OskariLayer> layers = getLayers(connection);
final List<OskariLayer> layers = getLayers(connection);
final int n = layers.size();

LOG.info("Start populating capabilities for layers - count:", layers.size());
LOG.info("Start populating capabilities for layers - count:", n);
Set<String> keys = new HashSet<>();
int progress = 0;
for(OskariLayer layer : layers) {
try {
final String layerKey = (layer.getSimplifiedUrl(true) + "----" + layer.getType()).toLowerCase();
if(keys.contains(layerKey)) {
progress++;
continue;
}
keys.add(layerKey);
final String xml = capabilitiesService.loadCapabilitiesFromService(layer, null);

OskariLayerCapabilities caps = capabilitiesService.createTemplate(layer);
caps.setData(xml);
insertCaps(connection, caps);
progress++;
LOG.info("Capabilities populated:", progress, "/", layers.size());
} catch (IOException e) {
// save empty result so we don't hang the system when having multiple layers from problematic service
OskariLayerCapabilities caps = capabilitiesService.createTemplate(layer);
caps.setData("");
insertCaps(connection, caps);
LOG.error(e, "Error getting capabilities for service", layer.getUrl());
for (int i = 0; i < n; i++) {
OskariLayer layer = layers.get(i);

String url = layer.getSimplifiedUrl(true);
String type = layer.getType();

String layerKey = url + "----" + layer.getType();
layerKey = layerKey.toLowerCase();
if (keys.contains(layerKey)) {
continue;
}

String data = "";
try {
data = CapabilitiesCacheService.loadCapabilitiesFromService(layer);
} catch (ServiceException e) {
LOG.error(e, "Error getting capabilities for service", url);
}

insertCaps(connection, type, url, data);

// Don't try the same service again (another layer might have same url + type)
keys.add(layerKey);

LOG.info("Capabilities populated:", i + 1, "/", n);
}
}

Expand All @@ -77,12 +72,12 @@ private List<OskariLayer> getLayers(Connection conn) throws SQLException {
return layers;
}

private void insertCaps(Connection conn, OskariLayerCapabilities caps) throws SQLException {
private void insertCaps(Connection conn, String type, String url, String data) throws SQLException {
final String sql = "INSERT INTO oskari_capabilities_cache (layertype, url, data) VALUES(?,?,?)";
try(PreparedStatement statement = conn.prepareStatement(sql)) {
statement.setString(1, caps.getLayertype());
statement.setString(2, caps.getUrl());
statement.setString(3, caps.getData());
statement.setString(1, type);
statement.setString(2, url);
statement.setString(3, data);
statement.execute();
}
}
Expand Down
6 changes: 3 additions & 3 deletions control-base/src/main/java/fi/nls/oskari/control/layer/GetWSCapabilitiesHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import fi.nls.oskari.wfs.GetGtWFSCapabilities;
import fi.nls.oskari.wms.GetGtWMSCapabilities;
import fi.nls.oskari.wmts.WMTSCapabilitiesParser;
import fi.nls.oskari.wmts.domain.WMTSCapabilities;
import org.json.JSONObject;

/**
Expand Down Expand Up @@ -60,8 +61,6 @@ public void handleAction(ActionParameters params) throws ActionException {
}
else {
if (OskariLayer.TYPE_WMTS.equals(layerType)) {
WMTSCapabilitiesParser parser = new WMTSCapabilitiesParser();

// setup capabilities URL
OskariLayerCapabilities caps = capabilitiesService.getCapabilities(url, OskariLayer.TYPE_WMTS, user, pw, version);
String capabilitiesXML = caps.getData();
Expand All @@ -70,7 +69,8 @@ public void handleAction(ActionParameters params) throws ActionException {
caps = capabilitiesService.getCapabilities(url, OskariLayer.TYPE_WMTS, user, pw, version, true);
capabilitiesXML = caps.getData();
}
JSONObject resultJSON = parser.parseCapabilitiesToJSON(capabilitiesXML, url, currentCrs);
WMTSCapabilities wmtsCaps = WMTSCapabilitiesParser.parseCapabilities(capabilitiesXML);
JSONObject resultJSON = WMTSCapabilitiesParser.asJSON(wmtsCaps, url, currentCrs);
JSONHelper.putValue(resultJSON, "xml", caps.getData());
ResponseHelper.writeResponse(params, resultJSON);
}
Expand Down
118 changes: 25 additions & 93 deletions control-base/src/main/java/fi/nls/oskari/control/layer/SaveLayerHandler.java
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package fi.nls.oskari.control.layer;

import fi.nls.oskari.service.capabilities.OskariLayerCapabilities;
import fi.mml.map.mapwindow.service.db.InspireThemeService;
import fi.mml.map.mapwindow.service.db.MaplayerProjectionService;
import fi.mml.map.mapwindow.service.wms.WebMapService;
import fi.mml.map.mapwindow.service.wms.WebMapServiceFactory;
import fi.mml.map.mapwindow.util.OskariLayerWorker;
import fi.mml.portti.domain.permissions.Permissions;
import fi.mml.portti.service.db.permissions.PermissionsService;
Expand All @@ -20,23 +20,17 @@
import fi.nls.oskari.map.data.domain.OskariLayerResource;
import fi.nls.oskari.map.layer.LayerGroupService;
import fi.nls.oskari.map.layer.OskariLayerService;
import fi.nls.oskari.map.layer.formatters.LayerJSONFormatterWMS;
import fi.nls.oskari.map.layer.formatters.LayerJSONFormatterWMTS;
import fi.nls.oskari.permission.domain.Permission;
import fi.nls.oskari.service.ServiceException;
import fi.nls.oskari.service.capabilities.CapabilitiesCacheService;
import fi.nls.oskari.service.capabilities.OskariLayerCapabilities;
import fi.nls.oskari.service.capabilities.OskariLayerCapabilitiesHelper;
import fi.nls.oskari.util.*;
import fi.nls.oskari.wfs.GetGtWFSCapabilities;
import fi.nls.oskari.wfs.WFSLayerConfigurationService;
import fi.nls.oskari.wfs.util.WFSParserConfigs;
import fi.nls.oskari.wmts.WMTSCapabilitiesParser;
import fi.nls.oskari.wmts.domain.ResourceUrl;
import fi.nls.oskari.wmts.domain.WMTSCapabilities;
import fi.nls.oskari.wmts.domain.WMTSCapabilitiesLayer;
import org.json.JSONArray;
import org.json.JSONObject;

import javax.servlet.http.HttpServletRequest;
import java.net.MalformedURLException;
import java.net.URL;
Expand Down Expand Up @@ -68,10 +62,6 @@ private class SaveResult {
private static final String PARAM_LAYER_URL = "layerUrl";
private static final String PARAM_SRS_NAME = "srs_name";

private static final String KEY_STYLES = "styles";
private static final String KEY_NAME = "name";


private static final String LAYER_NAME_PREFIX = "name_";
private static final String LAYER_TITLE_PREFIX = "title_";

Expand Down Expand Up @@ -269,11 +259,15 @@ private boolean handleRequestToMapLayer(final ActionParameters params, OskariLay
// get names and descriptions
final Enumeration<String> paramNames = request.getParameterNames();
while (paramNames.hasMoreElements()) {
String nextName = paramNames.nextElement();
if (nextName.indexOf(LAYER_NAME_PREFIX) == 0) {
ml.setName(nextName.substring(LAYER_NAME_PREFIX.length()).toLowerCase(), params.getHttpParam(nextName));
} else if (nextName.indexOf(LAYER_TITLE_PREFIX) == 0) {
ml.setTitle(nextName.substring(LAYER_TITLE_PREFIX.length()).toLowerCase(), params.getHttpParam(nextName));
String paramName = paramNames.nextElement();
if (paramName.startsWith(LAYER_NAME_PREFIX)) {
String lang = paramName.substring(LAYER_NAME_PREFIX.length()).toLowerCase();
String name = params.getHttpParam(paramName);
ml.setName(lang, name);
} else if (paramName.startsWith(LAYER_TITLE_PREFIX)) {
String lang = paramName.substring(LAYER_TITLE_PREFIX.length()).toLowerCase();
String title = params.getHttpParam(paramName);
ml.setTitle(lang, title);
}
}

Expand Down Expand Up @@ -488,8 +482,9 @@ private void validateInsertLayer(final ActionParameters params, OskariLayer ml)
}

}
private boolean handleWMSSpecific(final ActionParameters params, OskariLayer ml) throws ActionException {

private boolean handleWMSSpecific(final ActionParameters params, OskariLayer ml) {
// Do NOT modify the 'xslt' parameter
HttpServletRequest request = params.getRequest();
final String xslt = request.getParameter("xslt");
if(xslt != null) {
Expand All @@ -499,66 +494,23 @@ private boolean handleWMSSpecific(final ActionParameters params, OskariLayer ml)
ml.setGfiType(params.getHttpParam("gfiType", ml.getGfiType()));

try {
OskariLayerCapabilities capabilities = capabilitiesService.getCapabilities(ml, true);
// flush cache, otherwise only db is updated but code retains the old cached version
WebMapServiceFactory.flushCache(ml.getId());
// parse capabilities
WebMapService wms = WebMapServiceFactory.createFromXML(ml.getName(), capabilities.getData());
if (wms == null) {
throw new ServiceException("Couldn't parse capabilities for service!");
}
JSONObject caps = LayerJSONFormatterWMS.createCapabilitiesJSON(wms);
ml.setCapabilities(caps);
//TODO: similiar parsing for WMS GetCapabilities for admin layerselector and this
// Parsing is processed twice:
// 1st with geotools parsing for admin layerselector (styles are not parsered correct in all cases)
// 2nd in this class
// Fix default style, if no legendimage setup
String style = this.getDefaultStyle(ml, caps);
if (style != null) {
ml.setStyle(style);
}

ml.setSupportedCRSs(LayerJSONFormatterWMS.getCRSs(wms));


OskariLayerCapabilities raw = capabilitiesService.getCapabilities(ml, true);
WebMapService wms = OskariLayerCapabilitiesHelper.parseWMSCapabilities(raw.getData(), ml);
OskariLayerCapabilitiesHelper.setPropertiesFromCapabilitiesWMS(wms, ml);
return true;
} catch (ServiceException ex) {
LOG.error(ex, "Couldn't update capabilities for layer", ml);
return false;
}
}

private boolean handleWMTSSpecific(final ActionParameters params, OskariLayer ml) throws ActionException {

final String currentCrs = params.getHttpParam(PARAM_SRS_NAME, ml.getSrs_name());

private boolean handleWMTSSpecific(final ActionParameters params, OskariLayer ml) {
try {
OskariLayerCapabilities capabilities = capabilitiesService.getCapabilities(ml, true);
// flush cache, otherwise only db is updated but code retains the old cached version
WebMapServiceFactory.flushCache(ml.getId());
// parse capabilities
WMTSCapabilities caps = new WMTSCapabilitiesParser().parseCapabilities(capabilities.getData());
if (caps == null) {
throw new ServiceException("Couldn't parse capabilities for service!");
}
WMTSCapabilitiesLayer layer = caps.getLayer(ml.getName());
ResourceUrl resUrl = layer.getResourceUrlByType("tile");
if(resUrl != null) {
JSONHelper.putValue(ml.getOptions(), "requestEncoding", "REST");
JSONHelper.putValue(ml.getOptions(), "format", resUrl.getFormat());
JSONHelper.putValue(ml.getOptions(), "urlTemplate", resUrl.getTemplate());
}

JSONObject jscaps = LayerJSONFormatterWMTS.createCapabilitiesJSON(caps, layer);
ml.setCapabilities(jscaps);

ml.setTileMatrixSetId(LayerJSONFormatterWMTS.getTileMatrixSetId(jscaps, currentCrs));

ml.setSupportedCRSs(LayerJSONFormatterWMTS.getCRSs(caps, layer));

String currentCrs = params.getHttpParam(PARAM_SRS_NAME, ml.getSrs_name());
OskariLayerCapabilities raw = capabilitiesService.getCapabilities(ml, true);
WMTSCapabilities caps = WMTSCapabilitiesParser.parseCapabilities(raw.getData());
OskariLayerCapabilitiesHelper.setPropertiesFromCapabilitiesWMTS(caps, ml, currentCrs);
return true;

} catch (Exception ex) {
LOG.error(ex, "Couldn't update capabilities for layer", ml);
return false;
Expand All @@ -567,7 +519,7 @@ private boolean handleWMTSSpecific(final ActionParameters params, OskariLayer ml

private void handleWFSSpecific(final ActionParameters params, OskariLayer ml) throws ActionException {
// These are only in insert
ml.setSrs_name(params.getHttpParam("srs_name", ml.getSrs_name()));
ml.setSrs_name(params.getHttpParam(PARAM_SRS_NAME, ml.getSrs_name()));
ml.setVersion(params.getHttpParam("WFSVersion",ml.getVersion()));

// Put manual Refresh mode to attributes if true
Expand All @@ -585,10 +537,10 @@ private void handleWFSSpecific(final ActionParameters params, OskariLayer ml) th
ml.setAttributes(attributes);
}
// Get supported projections
Map<String, Object> capa = GetGtWFSCapabilities.getGtDataStoreCapabilities(ml.getUrl(), ml.getVersion(), ml.getUsername(), ml.getPassword(), ml.getSrs_name());
ml.setSupportedCRSs(GetGtWFSCapabilities.parseProjections(capa, ml.getVersion(), ml.getName()));

OskariLayerCapabilitiesHelper.setPropertiesFromCapabilitiesWFS(ml);
}


private String validateUrl(final String url) throws ActionParamsException {
try {
// check that it's a valid url by creating an URL object...
Expand All @@ -599,26 +551,6 @@ private String validateUrl(final String url) throws ActionParamsException {
return url;
}

/**
* Get 1st style name of capabilites styles
* @param ml layer data
* @param caps oskari wms capabilities
* @return
*/
private String getDefaultStyle(OskariLayer ml, final JSONObject caps) {
String style = null;
if (ml.getId() == -1 && ml.getLegendImage() == null && caps.has(KEY_STYLES)) {
// Take 1st style name for default - geotools parsing is not always correct
JSONArray styles = JSONHelper.getJSONArray(caps, KEY_STYLES);
JSONObject jstyle = JSONHelper.getJSONObject(styles, 0);
if (jstyle != null) {
style = JSONHelper.getStringFromJSON(jstyle, KEY_NAME, null);
return style;
}
}
return style;
}

private void addPermissionsForRoles(final OskariLayer ml, final User user, final String[] externalIds) {

OskariLayerResource res = new OskariLayerResource(ml);
Expand Down
21 changes: 9 additions & 12 deletions control-base/src/test/java/fi/nls/oskari/control/layer/CapabilitiesCacheServiceMock.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,29 +4,25 @@
import fi.nls.oskari.service.ServiceException;
import fi.nls.oskari.service.capabilities.CapabilitiesCacheService;
import fi.nls.oskari.service.capabilities.OskariLayerCapabilities;
import java.sql.Timestamp;

/**
* Created by SMAKINEN on 28.8.2015.
*/
public class CapabilitiesCacheServiceMock extends CapabilitiesCacheService {
private String response = null;

private final String response;

public CapabilitiesCacheServiceMock(final String response) {
this.response = response;
}

@Override
public OskariLayerCapabilities find(String url, String layertype, String version) {
OskariLayerCapabilities caps = new OskariLayerCapabilities();
caps.setUrl(url);
caps.setLayertype(layertype);
caps.setData(response);
caps.setVersion(version);
return caps;
final Timestamp ts = new Timestamp(System.currentTimeMillis());
return new OskariLayerCapabilities(10L, url, layertype, version, response, ts, ts);
}

@Override
public OskariLayerCapabilities save(OskariLayerCapabilities capabilities) {
return null;
public OskariLayerCapabilities save(OskariLayerCapabilities draft) {
return draft;
}

public OskariLayerCapabilities getCapabilities(OskariLayer layer) throws ServiceException {
Expand All @@ -35,4 +31,5 @@ public OskariLayerCapabilities getCapabilities(OskariLayer layer) throws Service
}
return super.getCapabilities(layer);
}

}
1 change: 1 addition & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1066,6 +1066,7 @@
<module>service-wfs</module>
<module>service-myplaces</module>
<module>service-rating</module>
<module>service-capabilities-update</module>

<module>service-search</module>
<module>service-search-nls</module>
Expand Down
4 changes: 4 additions & 0 deletions service-base/src/main/java/fi/nls/oskari/util/IOHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,10 @@ public static InputStream debugResponse(final InputStream in) throws IOException
return debug;
}

public static String getCharset(final HttpURLConnection con) {
return getCharset(con, null);
}

public static String getCharset(final HttpURLConnection con, final String defaultCharset) {
final String contentType = con.getContentType();
final String[] values = contentType.split(";");
Expand Down
Loading

0 comments on commit 104702c

Please sign in to comment.