added ensemble percentage table

Author: ckrew

tethysapp/streamflow_prediction_tool/app.py

@@ -49,6 +49,11 @@ def url_maps(self):
                         'get-ecmwf-hydrograph-plot',
                     controller='streamflow_prediction_tool.controllers_ajax'
                                '.get_ecmwf_hydrograph_plot'),
+            url_map(name='get_ecmwf_forecast_probabilities',
+                    url='streamflow-prediction-tool/map/'
+                        'get-ecmwf-forecast-probabilities',
+                    controller='streamflow_prediction_tool.controllers_ajax'
+                               '.get_ecmwf_forecast_probabilities'),
             url_map(name='get_return_periods_ajax',
                     url='streamflow-prediction-tool/map/'
                         'get-return-periods',
@@ -226,10 +231,6 @@ def url_maps(self):
                     url='streamflow-prediction-tool/api/GetForecast',
                     controller='streamflow_prediction_tool.controllers_api'
                                '.get_ecmwf_forecast'),
-            url_map(name='waterml',
-                    url='streamflow-prediction-tool/api/GetEnsemble',
-                    controller='streamflow_prediction_tool.controllers_api'
-                               '.get_ecmwf_ensemble_csv'),
             url_map(name='era_interim',
                     url='streamflow-prediction-tool/api/GetHistoricData',
                     controller='streamflow_prediction_tool.controllers_api'

tethysapp/streamflow_prediction_tool/controllers_ajax.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 """controllers_ajax.py
-
     Created by Alan D. Snow, Curtis Rae, Shawn Crawley 2015.
     License: BSD 3-Clause
 """
@@ -18,6 +17,8 @@
 from sqlalchemy.orm.exc import ObjectDeletedError
 import xarray
 
+from .functions import ecmwf_find_most_current_files
+
 # django imports
 from django.contrib.auth.decorators import user_passes_test, login_required
 from django.core.exceptions import PermissionDenied
@@ -45,7 +46,8 @@
                                     get_return_period_dict,
                                     get_return_period_ploty_info)
 from .controllers_validators import (validate_historical_data,
-                                     validate_watershed_info)
+                                     validate_watershed_info,
+                                     validate_rivid_info)
 from .functions import (delete_from_database,
                         format_name,
                         get_units_title,
@@ -408,6 +410,7 @@ def get_ecmwf_hydrograph_plot(request):
     forecast_statistics, watershed_name, subbasin_name, river_id, units = \
         get_ecmwf_forecast_statistics(request)
 
+
     # ensure lower std dev values limited by the min
     std_dev_lower_df = \
         forecast_statistics['std_dev_range_lower']
@@ -953,7 +956,6 @@ def get_monthly_seasonal_streamflow_chart(request):
 def get_flow_duration_curve(request):
     """
     Generate flow duration curve for hydrologic time series data
-
     Based on: http://earthpy.org/flow.html
     """
     historical_data_file, river_id, watershed_name, subbasin_name = \
@@ -1800,3 +1802,89 @@ def watershed_group_update(request):
     return JsonResponse({
         'success': "Watershed group successfully updated."
     })
+
+
+@require_GET
+@login_required
+@exceptions_to_http_status
+def get_ecmwf_forecast_probabilities(request):
+    """
+    Returns the statistics for the 52 member forecast
+    """
+
+    path_to_rapid_output = app.get_custom_setting('ecmwf_forecast_folder')
+    if not os.path.exists(path_to_rapid_output):
+        raise SettingsError('Location of ECMWF forecast files faulty. '
+                            'Please check settings.')
+
+    # get/check information from AJAX request
+    get_info = request.GET
+    watershed_name, subbasin_name = validate_watershed_info(get_info)
+    river_id = validate_rivid_info(get_info)
+
+    forecast_folder = get_info.get('forecast_folder')
+    if not forecast_folder:
+        forecast_folder = 'most_recent'
+
+    # find/check current output datasets
+    path_to_output_files = \
+        os.path.join(path_to_rapid_output,
+                     "{0}-{1}".format(watershed_name, subbasin_name))
+    forecast_nc_list, start_date = \
+        ecmwf_find_most_current_files(path_to_output_files, forecast_folder)
+    if not forecast_nc_list or not start_date:
+        raise NotFoundError('ECMWF forecast for %s (%s).'
+                            % (watershed_name, subbasin_name))
+
+    # combine 52 ensembles
+    qout_datasets = []
+    ensemble_index_list = []
+    with rivid_exception_handler("ECMWF Forecast", river_id):
+        for forecast_nc in forecast_nc_list:
+            if forecast_nc.endswith("52.nc"):
+                continue
+            else:
+                ensemble_index_list.append(
+                    int(os.path.basename(forecast_nc)[:-3].split("_")[-1])
+                )
+                qout_datasets.append(
+                    xarray.open_dataset(forecast_nc, autoclose=True)
+                          .sel(rivid=river_id).Qout
+                )
+
+    merged_ds = xarray.concat(qout_datasets,
+                              pd.Index(ensemble_index_list, name='ensemble'))
+
+    returnperiods = {}
+
+    return_period_data = get_return_period_dict(request)
+
+    returnperiods['two'] = float(return_period_data["two"])
+    returnperiods['ten'] = float(return_period_data["ten"])
+    returnperiods['twenty'] = float(return_period_data["twenty"])
+
+    timelist = merged_ds.time.values
+    datelist = []
+    shortdate = []
+    problist = {'two':[],'ten':[],'twenty':[]}
+
+    for timestep in timelist:
+        x = pd.to_datetime(timestep)
+        if str(x.date()) not in datelist:
+            datelist.append(str(x.date()))
+            shortdate.append(str(x.date())[-5:])
+
+    for date in datelist:
+        mergedtime = merged_ds.sel(time=date)
+        for period in returnperiods:
+            retperflow = float(returnperiods[period])
+            probabilities = xarray.where(mergedtime > retperflow, 1, 0).sum(dim='ensemble').astype(float)/51.0*100.
+            probability = probabilities.max().round(2).values
+            problist[period].append((date,np.asscalar(probability)))
+
+    problist['dates'] = shortdate
+
+    for entries in problist:
+        problist[entries] = problist[entries][:-5 or None]
+
+    return JsonResponse({'success':True, 'percentages':problist})

tethysapp/streamflow_prediction_tool/controllers_api.py

@@ -4,9 +4,9 @@
     Author: Michael Suffront & Alan D. Snow, 2017
     License: BSD 3-Clause
 """
-from django.http import JsonResponse, HttpResponse
+from django.http import JsonResponse
 from django.shortcuts import render_to_response
-from rest_framework.authentication import TokenAuthentication, SessionAuthentication
+from rest_framework.authentication import TokenAuthentication
 from rest_framework.decorators import api_view, authentication_classes
 
 from .app import StreamflowPredictionTool as app
@@ -15,20 +15,16 @@
                                generate_warning_points)
 from .controllers_functions import (get_ecmwf_avaialable_dates,
                                     get_ecmwf_forecast_statistics,
-                                    get_ecmwf_ensemble,
                                     get_historic_streamflow_series,
                                     get_return_period_dict)
 from .controllers_validators import validate_historical_data
 from .exception_handling import InvalidData, exceptions_to_http_status
 from .functions import get_units_title
 from .model import Watershed
 
-import pandas as pd
-from csv import writer as csv_writer
-
 
 @api_view(['GET'])
-@authentication_classes((TokenAuthentication, SessionAuthentication,))
+@authentication_classes((TokenAuthentication,))
 @exceptions_to_http_status
 def get_ecmwf_forecast(request):
     """
@@ -101,40 +97,7 @@ def get_ecmwf_forecast(request):
 
 
 @api_view(['GET'])
-@authentication_classes((TokenAuthentication, SessionAuthentication,))
-@exceptions_to_http_status
-def get_ecmwf_ensemble_csv(request):
-    """
-    Retrieve the forecasted streamflow as CSV
-    """
-    # retrieve statistics
-    forecast_statistics, watershed_name, subbasin_name, river_id, units = \
-        get_ecmwf_ensemble(request)
-
-    # prepare to write response for CSV
-    response = HttpResponse(content_type='text/csv')
-    response['Content-Disposition'] = \
-        'attachment; filename=forecasted_ensembles_{0}_{1}_{2}.csv' \
-        .format(watershed_name,
-                subbasin_name,
-                river_id)
-
-    writer = csv_writer(response)
-    forecast_df = pd.DataFrame(forecast_statistics)
-    column_names = (forecast_df.columns.values +
-                    [' ({}3/s)'.format(get_units_title(units))]
-                    ).tolist()
-
-    writer.writerow(['datetime'] + column_names)
-
-    for row_data in forecast_df.itertuples():
-        writer.writerow(row_data)
-
-    return response
-
-
-@api_view(['GET'])
-@authentication_classes((TokenAuthentication, SessionAuthentication,))
+@authentication_classes((TokenAuthentication,))
 @exceptions_to_http_status
 def get_historic_data(request):
     """
@@ -190,7 +153,7 @@ def get_historic_data(request):
 
 
 @api_view(['GET'])
-@authentication_classes((TokenAuthentication, SessionAuthentication,))
+@authentication_classes((TokenAuthentication,))
 @exceptions_to_http_status
 def get_return_periods_api(request):
     """
@@ -200,7 +163,7 @@ def get_return_periods_api(request):
 
 
 @api_view(['GET'])
-@authentication_classes((TokenAuthentication, SessionAuthentication,))
+@authentication_classes((TokenAuthentication,))
 @exceptions_to_http_status
 def get_available_dates(request):
     """
@@ -217,7 +180,7 @@ def get_available_dates(request):
 
 
 @api_view(['GET'])
-@authentication_classes((TokenAuthentication, SessionAuthentication,))
+@authentication_classes((TokenAuthentication,))
 @exceptions_to_http_status
 def get_watershed_list(request):  # pylint: disable=unused-argument
     """
@@ -248,7 +211,7 @@ def get_watershed_list(request):  # pylint: disable=unused-argument
 
 
 @api_view(['GET'])
-@authentication_classes((TokenAuthentication, SessionAuthentication,))
+@authentication_classes((TokenAuthentication,))
 @exceptions_to_http_status
 def get_warning_points(request):
     """