Calculates 4 statistics, how many threshold exceedences are: forecast but not observed, observed but not forecast, forecast but not within an acceptable time range and for how many are correctly forecast. For the calculation input forecast time series with value properties are compared to observed time series.

Threshold Exceedance Count config example
<?xml version="1.0" encoding="UTF-8"?>
<!-- edited with XMLSpy v2007 sp2 (http://www.altova.com) by WL | Delft Hydraulics (WL | Delft Hydraulics) -->
<transformationModule version="1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://www.wldelft.nl/fews"
                 xsi:schemaLocation="http://www.wldelft.nl/fews http://fews.wldelft.nl/schemas/version1.0/transformationModule.xsd">
   <variable>
      <variableId>inputObserved</variableId>
      <timeSeriesSet>
         <moduleInstanceId>ThresholdExceedanceCountTest</moduleInstanceId>
         <valueType>scalar</valueType>
         <parameterId>H.obs.proc</parameterId>
         <locationSetId>ThresholdExceedanceCountLocationSet</locationSetId>
         <timeSeriesType>external forecasting</timeSeriesType>
         <timeStep unit="minute" multiplier="15"/>
         <readWriteMode>read complete forecast</readWriteMode>
      </timeSeriesSet>
   </variable>
   <variable>
      <variableId>inputForecast</variableId>
      <timeSeriesSet>
         <moduleInstanceId>ThresholdExceedanceCountTest</moduleInstanceId>
         <valueType>scalar</valueType>
         <parameterId>H.issued.fcst</parameterId>
         <locationSetId>ThresholdExceedanceCountLocationSet</locationSetId>
         <timeSeriesType>external forecasting</timeSeriesType>
         <timeStep unit="nonequidistant"/>
         <readWriteMode>read complete forecast</readWriteMode>
      </timeSeriesSet>
   </variable>
   <variable>
      <variableId>inputForecastSequence</variableId>
      <timeSeriesSet>
         <moduleInstanceId>ThresholdExceedanceCountTest</moduleInstanceId>
         <valueType>scalar</valueType>
         <parameterId>Warning</parameterId>
         <!--<locationRelationId>WARNING_ID</locationRelationId>-->
         <locationSetId>ThresholdExceedanceCountLocationSet</locationSetId>
         <timeSeriesType>external forecasting</timeSeriesType>
         <timeStep unit="nonequidistant"/>
         <readWriteMode>read complete forecast</readWriteMode>
      </timeSeriesSet>
   </variable>
   <variable>
      <variableId>falseAlarm</variableId>
      <timeSeriesSet>
         <moduleInstanceId>ThresholdExceedanceCountTest</moduleInstanceId>
         <valueType>scalar</valueType>
         <parameterId>ValueCount</parameterId>
         <qualifierId>falseAlarm</qualifierId>
         <locationSetId>ThresholdExceedanceCountLocationSet</locationSetId>
         <timeSeriesType>external historical</timeSeriesType>
         <timeStep unit="nonequidistant"/>
         <relativeViewPeriod unit="day" start="-1" end="1"/>
         <readWriteMode>editing visible to all future task runs</readWriteMode>
      </timeSeriesSet>
   </variable>
   <variable>
      <variableId>missedCrossings</variableId>
      <timeSeriesSet>
         <moduleInstanceId>ThresholdExceedanceCountTest</moduleInstanceId>
         <valueType>scalar</valueType>
         <parameterId>ValueCount</parameterId>
         <qualifierId>missedCrossing</qualifierId>
         <locationSetId>ThresholdExceedanceCountLocationSet</locationSetId>
         <timeSeriesType>external historical</timeSeriesType>
         <timeStep unit="nonequidistant"/>
         <relativeViewPeriod unit="day" start="-1" end="1"/>
         <readWriteMode>editing visible to all future task runs</readWriteMode>
      </timeSeriesSet>
   </variable>
   <variable>
      <variableId>missedTime</variableId>
      <timeSeriesSet>
         <moduleInstanceId>ThresholdExceedanceCountTest</moduleInstanceId>
         <valueType>scalar</valueType>
         <parameterId>ValueCount</parameterId>
         <qualifierId>missedTime</qualifierId>
         <locationSetId>ThresholdExceedanceCountLocationSet</locationSetId>
         <timeSeriesType>external historical</timeSeriesType>
         <timeStep unit="nonequidistant"/>
         <relativeViewPeriod unit="day" start="-1" end="1"/>
         <readWriteMode>editing visible to all future task runs</readWriteMode>
      </timeSeriesSet>
   </variable>
   <variable>
      <variableId>hit</variableId>
      <timeSeriesSet>
         <moduleInstanceId>ThresholdExceedanceCountTest</moduleInstanceId>
         <valueType>scalar</valueType>
         <parameterId>ValueCount</parameterId>
         <qualifierId>hit</qualifierId>
         <locationSetId>ThresholdExceedanceCountLocationSet</locationSetId>
         <timeSeriesType>external historical</timeSeriesType>
         <timeStep unit="nonequidistant"/>
         <relativeViewPeriod unit="day" start="-1" end="1"/>
         <readWriteMode>editing visible to all future task runs</readWriteMode>
      </timeSeriesSet>
   </variable>
   <!--<transformation id="ThresholdExceedanceTest" useRelatedInputLocationForOutputMatching="false">-->
   <transformation id="ThresholdExceedanceTest">
      <statisticsValueProperties>
         <thresholdExceedanceCount>
            <inputObservedVariable>
               <variableId>inputObserved</variableId>
            </inputObservedVariable>
            <inputForecastVariable>
               <variableId>inputForecast</variableId>
            </inputForecastVariable>
            <inputForecastSequenceVariable>
               <variableId>inputForecastSequence</variableId>
            </inputForecastSequenceVariable>
            <thresholdCrossing valueAttributeId="thresholdValueAAA" leadTimeAttributeId="thresholdLeadTimeAAA"/>
            <thresholdCrossing valueAttributeId="thresholdValueBBB" leadTimeAttributeId="thresholdLeadTimeBBB"/>
            <thresholdCrossing valueAttributeId="thresholdValueCCC" leadTimeAttributeId="thresholdLeadTimeCCC"/>
            <forecastSearchPeriod start="-30" end="0" unit="day"/>
            <startSequenceValueProperty id="Phase" value="NEW"/>
            <endSequenceValueProperty id="Phase" value="FIN"/>
            <outputFalseAlarmCountVariable>
               <variableId>falseAlarm</variableId>
            </outputFalseAlarmCountVariable>
            <outputMissedCrossingsCountVariable>
               <variableId>missedCrossings</variableId>
            </outputMissedCrossingsCountVariable>
            <outputMissedTimeCountVariable>
               <variableId>missedTime</variableId>
            </outputMissedTimeCountVariable>
            <outputHitCountVariable>
               <variableId>hit</variableId>
            </outputHitCountVariable>
         </thresholdExceedanceCount>
      </statisticsValueProperties>
   </transformation>
</transformationModule>

There are 3 different inputs

<inputObservedVariable> which contains the observed time series from which the threshold crossings are taken
<inputForecastVariable> which contains the forecasts that predicted threshold crossings
<inputForecastSequenceVariable> which contains forecasts that determine a sequence of periods in which forecasts and crossings should be within in order to be matched to eachother. 
The sequence periods are determined via <startSequenceValueProperty .../> and <endSequenceValueProperty .../> and their value property id's and values.

For <startSequenceValueProperty id="Phase" value="NEW"/> <endSequenceValueProperty id="Phase" value="FIN"/> sequence periods will be from the forecast series with valueProperty "Phase" = "NEW" until a forecast series with "Phase" = "FIN" and then new periods will start from the next "NEW" until the next "FIN" and so on.


There can be multiple threshold crossings for each location, based on <thresholdCrossing .../> elements 
they refer to an attribute for the height of the threshold and and attribute for the time which the forecast should be before in order for the forecast to be considered "on time".
For locations which do not have a value for one of the attributes the thresholds are skipped.


For each threshold the first crossing through its threshold value within its sequence period, will be matched to the first forecast that predicted the threshold crossing within the sequence period.

If the forecast was sent before the actual crossing minus the lead time, the forecast will be counted as correct which counts as a value +1 for the <outputHitCountVariable>.

If the forecast was sent after the actual crossing minus the lead time, the forecast will be counted as missed time which means a value +1 for the <outputMissedTimeCountVariable>.

If the forecast was sent even after the actual crossing (so not even without minus the lead time), the forecast will be counted as missed height which means a value +1 for the <outputMissedHeightCountVariable>.

If the forecast was never matched with a crossing, the forecast will be counted as false alarm which means a value +1 for the <outputFalseAlarmCountVariable>.

For all the crossings that are never matched with a forecast will also be counted as missed height which means a value +1 for the <outputMissedHeightCountVariable>.


Split between matching and evaluation

Since 2021.02 this transformation has been split in 2 steps:

  1. where the observed crossings are matched with the first forecast that predicted it. For each match the observed crossing is written to a time stamp property called Prediction_Value_Exceed_Time.
  2. the hits, missed time, missed height and false alarms are counted based on the (in step 1) written time stamp value property Prediction_Value_Exceed_Time


Config example step 1

Threshold Exceedance Matching
<transformation id="ThresholdExceedanceTest">
   <statisticsValueProperties>
      <thresholdExceedanceMatching>
         <inputObservedVariable>
            <variableId>inputObserved</variableId>
         </inputObservedVariable>
         <inputForecastVariable>
            <variableId>inputForecast</variableId>
         </inputForecastVariable>
         <inputForecastSequenceVariable>
            <variableId>inputForecastSequence</variableId>
         </inputForecastSequenceVariable>
         <thresholdCrossing valueAttributeId="thresholdValueAAA" leadTimeAttributeId="thresholdLeadTimeAAA"/>
         <thresholdCrossing valueAttributeId="thresholdValueBBB" leadTimeAttributeId="thresholdLeadTimeBBB"/>
         <thresholdCrossing valueAttributeId="thresholdValueCCC" leadTimeAttributeId="thresholdLeadTimeCCC"/>
         <forecastSearchPeriod start="-30" end="0" unit="day"/>
         <startSequenceValueProperty id="Phase" value="NEW"/>
         <endSequenceValueProperty id="Phase" value="FIN"/>
         <forecastSelectValueProperty id="Hydrograph_Type" value="Reach"/>
         <forecastSelectValueProperty id="Hydrograph_Type" value="Peak"/>
         <forecastSelectValueProperty id="Hydrograph_Type" value="Exceed"/>
         <outputVariable>
            <variableId>outputForecast</variableId>
         </outputVariable>
      </thresholdExceedanceMatching>
   </statisticsValueProperties>
</transformation>

Config example step 2

Threshold Exceedance Evaluation
<transformation id="ThresholdExceedanceTest">
   <statisticsValueProperties>
      <thresholdExceedanceEvaluation>
         <inputObservedVariable>
            <variableId>inputObserved</variableId>
         </inputObservedVariable>
         <inputForecastVariable>
            <variableId>inputForecast</variableId>
         </inputForecastVariable>
         <inputForecastSequenceVariable>
            <variableId>inputForecastSequence</variableId>
         </inputForecastSequenceVariable>
         <thresholdCrossing valueAttributeId="thresholdValueAAA" leadTimeAttributeId="thresholdLeadTimeAAA"/>
         <thresholdCrossing valueAttributeId="thresholdValueBBB" leadTimeAttributeId="thresholdLeadTimeBBB"/>
         <thresholdCrossing valueAttributeId="thresholdValueCCC" leadTimeAttributeId="thresholdLeadTimeCCC"/>
         <forecastSearchPeriod start="-30" end="0" unit="day"/>
         <startSequenceValueProperty id="Phase" value="NEW"/>
         <endSequenceValueProperty id="Phase" value="FIN"/>
         <forecastSelectValueProperty id="Hydrograph_Type" value="Reach"/>
         <forecastSelectValueProperty id="Hydrograph_Type" value="Peak"/>
         <forecastSelectValueProperty id="Hydrograph_Type" value="Exceed"/>
         <outputFalseAlarmCountVariable>
            <variableId>falseAlarm</variableId>
         </outputFalseAlarmCountVariable>
         <outputMissedCrossingsCountVariable>
            <variableId>missedCrossings</variableId>
         </outputMissedCrossingsCountVariable>
         <outputMissedTimeCountVariable>
            <variableId>missedTime</variableId>
         </outputMissedTimeCountVariable>
         <outputHitCountVariable>
            <variableId>hit</variableId>
         </outputHitCountVariable>
      </thresholdExceedanceEvaluation>
   </statisticsValueProperties>
</transformation>
  • No labels

Disclaimer