Skip to content

Commit

Permalink
thredds: triple max memory
Browse files Browse the repository at this point in the history 
This commit is for Ouranos production only.

Tried previously to double the memory and the problems comes back again.

Seeing these timed out errors in `docker logs proxy`:

```
2022/02/15 20:26:22 [error] 12#12: *878921 upstream timed out (110: Connection timed out) while reading upstream, client: 204.19.71.2, server: localho
st, request: "GET /twitcher/ows/proxy/thredds/dodsC/datasets/simulations/climex/day_climex-crcm5_historical+rcp85.ncml.dods?pr.pr%5b0:49%5d%5b17190:17
219%5d%5b0:279%5d%5b0:279%5d HTTP/1.1", upstream: "http://172.16.0.20:8000/ows/proxy/thredds/dodsC/datasets/simulations/climex/day_climex-crcm5_histor
ical+rcp85.ncml.dods?pr.pr%5b0:49%5d%5b17190:17219%5d%5b0:279%5d%5b0:279%5d", host: "pavics.ouranos.ca"
```

`docker logs thredds` shows:

```
Table PsuedoStructure(time) featureType POINT: lat/lon/time coord not found
<?xml version="1.0" encoding="UTF-8"?>^M
<featureDataset location="/pavics-data/nrcan/nrcan_northamerica_monthly/cmi/nrcan_northamerica_monthly_1973_cmi.nc">^M
  <analyser class="ucar.nc2.ft.point.standard.plug.CFpointObs" />^M
  <table name="PsuedoStructure(time)" featureType="POINT" class="class ucar.nc2.ft.point.standard.Table$TableStructure">^M
    <coordinate kind="time">time</coordinate>^M
    <variable>cmi</variable>^M
  </table>^M
</featureDataset>^M
Failed (wrapUnknown) to find Datatype Factory for= /pavics-data/nrcan/nrcan_northamerica_monthly/cmi/nrcan_northamerica_monthly_1973_cmi.nc
Table PsuedoStructure(time) featureType POINT: lat/lon/time coord not found
<?xml version="1.0" encoding="UTF-8"?>^M
<featureDataset location="/pavics-data/wps_outputs/a0f4e60d-f55f-11eb-8bbc-8f3ed51e594b/nrcan_canada_daily_tasmin_1990_avg.nc">^M
  <analyser class="Default" />^M
  <table name="PsuedoStructure(time)" class="class ucar.nc2.ft.point.standard.Table$TableStructure">^M
    <coordinate kind="lat">lat</coordinate>^M
    <coordinate kind="lon">lon</coordinate>^M
    <coordinate kind="time">time</coordinate>^M
    <variable>tasmin</variable>^M
  </table>^M
</featureDataset>^M
Failed (wrapUnknown) to find Datatype Factory for= /pavics-data/wps_outputs/a0f4e60d-f55f-11eb-8bbc-8f3ed51e594b/nrcan_canada_daily_tasmin_1990_avg.nc
Table PsuedoStructure(time) featureType POINT: lat/lon/time coord not found
<?xml version="1.0" encoding="UTF-8"?>^M
<featureDataset location="/pavics-data/caspar/daily/GEPS_20180526.nc">^M
  <analyser class="Default" />^M
  <table name="PsuedoStructure(time)" class="class ucar.nc2.ft.point.standard.Table$TableStructure">^M
    <coordinate kind="lat">lat</coordinate>^M
    <coordinate kind="lon">lon</coordinate>^M
    <coordinate kind="time">time</coordinate>^M
  </table>^M
</featureDataset>^M
Failed (wrapUnknown) to find Datatype Factory for= /pavics-data/caspar/daily/GEPS_20180526.nc
java.lang.IllegalStateException: NcSDArray InvalidRangeException=Illegal Range for dimension 0: last requested 54359 > max 53999
        at thredds.server.opendap.NcSDArray.read(NcSDArray.java:137)
        at thredds.server.opendap.NcSDGrid.read(NcSDGrid.java:73)
        at opendap.servers.SDGrid.serialize(SDGrid.java:415)
        at opendap.servers.CEEvaluator.send(CEEvaluator.java:315)
        at thredds.server.opendap.OpendapServlet.doGetDAP2Data(OpendapServlet.java:531)
        at thredds.server.opendap.OpendapServlet.doGet(OpendapServlet.java:218)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:655)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:764)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at thredds.servlet.filter.CookieFilter.doFilter(CookieFilter.java:55)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at thredds.servlet.filter.RequestQueryFilter.doFilter(RequestQueryFilter.java:118)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at org.apache.catalina.filters.CorsFilter.handleNonCORS(CorsFilter.java:368)
        at org.apache.catalina.filters.CorsFilter.doFilter(CorsFilter.java:174)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at org.apache.catalina.filters.HttpHeaderSecurityFilter.doFilter(HttpHeaderSecurityFilter.java:126)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at org.apache.logging.log4j.web.Log4jServletFilter.doFilter(Log4jServletFilter.java:71)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at thredds.servlet.filter.RequestCORSFilter.doFilterInternal(RequestCORSFilter.java:49)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
        at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347)
        at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at thredds.servlet.filter.RequestPathFilter.doFilter(RequestPathFilter.java:94)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at thredds.server.RequestBracketingLogMessageFilter.doFilter(RequestBracketingLogMessageFilter.java:81)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:196)
        at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:97)
        at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:542)
        at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:135)
        at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:81)
        at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:698)
        at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:78)
        at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:364)
        at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:624)
        at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
        at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:831)
        at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1673)
        at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
        at org.apache.tomcat.util.threads.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1191)
        at org.apache.tomcat.util.threads.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:659)
        at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
        at java.lang.Thread.run(Thread.java:748)

java.lang.OutOfMemoryError: Java heap space
        at ucar.ma2.ArrayFloat.<init>(ArrayFloat.java:100)
        at ucar.ma2.ArrayFloat$D4.<init>(ArrayFloat.java:335)
        at ucar.ma2.ArrayFloat$D4.<init>(ArrayFloat.java:327)
        at ucar.ma2.ArrayFloat.factory(ArrayFloat.java:65)
        at ucar.ma2.ArrayFloat.factory(ArrayFloat.java:50)
        at ucar.ma2.Array.factory(Array.java:112)
        at ucar.ma2.Array.factory(Array.java:92)
        at ucar.nc2.ncml.AggregationOuterDimension.reallyRead(AggregationOuterDimension.java:242)
        at ucar.nc2.dataset.VariableDS._read(VariableDS.java:537)
        at ucar.nc2.Variable.read(Variable.java:709)
        at ucar.nc2.Variable.read(Variable.java:683)
        at thredds.server.opendap.NcSDArray.read(NcSDArray.java:115)
        at thredds.server.opendap.NcSDGrid.read(NcSDGrid.java:73)
        at opendap.servers.SDGrid.serialize(SDGrid.java:415)
        at opendap.servers.CEEvaluator.send(CEEvaluator.java:315)
        at thredds.server.opendap.OpendapServlet.doGetDAP2Data(OpendapServlet.java:531)
        at thredds.server.opendap.OpendapServlet.doGet(OpendapServlet.java:218)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:655)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:764)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at thredds.servlet.filter.CookieFilter.doFilter(CookieFilter.java:55)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at thredds.servlet.filter.RequestQueryFilter.doFilter(RequestQueryFilter.java:118)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at org.apache.catalina.filters.CorsFilter.handleNonCORS(CorsFilter.java:368)
        at org.apache.catalina.filters.CorsFilter.doFilter(CorsFilter.java:174)
```
  • Loading branch information
@tlvu
tlvu committed Feb 15, 2022
1 parent 571811f commit 76dd3c8
Showing 1 changed file with 1 addition and 1 deletion.
2 changes: 1 addition & 1 deletion birdhouse/config/thredds/thredds.env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ TDS_CONTENT_ROOT_PATH=/usr/local/tomcat/content

# The minimum and maximum Java heap space memory to be allocated to the TDS

THREDDS_XMX_SIZE=4G
THREDDS_XMX_SIZE=12G

THREDDS_XMS_SIZE=1G

Expand Down

0 comments on commit 76dd3c8

Please sign in to comment.