Merge with current version -- HJB

Merge remote-tracking branch 'origin/master' into NXmx_multimodule_and_dectris_changes

.travis.yml

@@ -0,0 +1,18 @@
+# :file: .travis.yml
+#
+# :url:  https://travis-ci.org/nexusformat/definitions
+#
+# for advice, see:
+#  https://docs.travis-ci.com/user/customizing-the-build#The-Build-Lifecycle
+
+language: python
+python:
+  - "2.7"
+  - "3.5"
+
+install:
+  - pip install lxml six
+
+# command to run tests
+script:
+  - python ./utils/test_nxdl.py

Makefile

@@ -33,6 +33,10 @@ builddir ::
 makebuilddir :: builddir
 	$(MAKE) -C build
 
+remakebuilddir ::
+	python utils/build_preparation.py . build
+	$(MAKE) -C build
+
 cleanbuilddir ::
 	$(MAKE) -C build clean
 

NXDL_VERSION

@@ -1 +1 @@
-3.1
+3.2

README.md

@@ -1,6 +1,7 @@
 * NeXus: http://www.nexusformat.org/
 * documentation: http://download.nexusformat.org/doc/html/index.html
 * build server: http://build.nexusformat.org/
+* travis-ci: syntax check of every NXDL file [![Build Status](https://travis-ci.org/nexusformat/definitions.svg)](https://travis-ci.org/nexusformat/definitions)
 
 These are the components that define the structure of NeXus data files 
 in the development directory.
@@ -17,6 +18,7 @@ contributed_definitions/    | NXDL files from the community
 impatient-guide/            | *NeXus for the Impatient*
 jenkins_build               | configuration for Jenkins continuous integration service
 manual/                     | Sphinx source files for the NeXus documentation
+manual_archive/             | historical copy of the NeXus documentation
 nxdl.xsd                    | XML Schema for NXDL files
 nxdlTypes.xsd               | called by nxdl.xsd
 package/                    | directory for packaging this content

base_classes/NXaperture.nxdl.xml

@@ -53,5 +53,5 @@
     <field name="description">
         <doc>Description of aperture</doc>
     </field>
-    <group type="NXnote"><doc>describe an additional information in a note*</doc></group>
+    <group type="NXnote"><doc>describe any additional information in a note*</doc></group>
 </definition>

base_classes/NXdata.nxdl.xml

@@ -147,7 +147,7 @@
 	</attribute>
 
 	<doc>
-		(**required**) :ref:`NXdata` describes the plottable data and related dimension scales. 
+		:ref:`NXdata` describes the plottable data and related dimension scales. 
 
 		.. index:: plotting
 
@@ -337,9 +337,11 @@
 
 				Examples::
 
-					@I_uncertainties="Idev"
+					@uncertainties="data_errors"
+
+					@uncertainties="Idev"
 
-					@Q_uncertainties="dQw", "dQl"		
+					@uncertainties="dQw", "dQl"		
 
 			</doc>
 		</attribute>

base_classes/NXentry.nxdl.xml

@@ -51,12 +51,39 @@
 		(**required**) :ref:`NXentry` describes the measurement. 
 
 		The top-level NeXus group which contains all the data and associated
-		information that comprise a single measurement. It is mandatory that there is at least one
-		group of this type in the NeXus file.
-	</doc>
+		information that comprise a single measurement. 
+		It is mandatory that there is at least one
+		group of this type in the NeXus file.	</doc>
 
-	<group type="NXdata" minOccurs="1">
-		<doc>The required data group</doc>
+	<group type="NXdata">
+		<doc>
+			The data group
+
+			.. note:: Before the NIAC2016 meeting [#], at least one
+			   :ref:`NXdata` group was required in each :ref:`NXentry` group.
+			   At the NIAC2016 meeting, it was decided to make :ref:`NXdata` 
+			   an optional group in :ref:`NXentry` groups for data files that 
+			   do not use an application definition.
+			   It is recommended strongly that all NeXus data files provide 
+			   a NXdata group.  
+			   It is permissable to omit the NXdata group only when 
+			   defining the default plot is not practical or possible
+			   from the available data.
+
+			   For example, neutron event data may not have anything that 
+			   makes a useful plot without extensive processing.
+
+			   Certain application definitions override this decision and
+			   require an :ref:`NXdata` group
+			   in the :ref:`NXentry` group.  The ``minOccurs=0`` attribute
+			   in the application definition will indicate the 
+			   :ref:`NXdata` group
+			   is optional, otherwise, it is required.
+
+			.. [#] NIAC2016: http://wiki.nexusformat.org/NIAC2016,
+			   https://github.com/nexusformat/NIAC/issues/16
+
+		</doc>
 	</group>
 
 	<attribute name="IDF_Version">

base_classes/NXevent_data.nxdl.xml

@@ -25,26 +25,50 @@
 	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 	xsi:schemaLocation="http://definition.nexusformat.org/nxdl/3.1 ../nxdl.xsd"
 	name="NXevent_data" 
-	version="1.0"
+	version="1.1"
 	type="group" extends="NXobject">
-	<doc>Time-of-flight events</doc>
-	<field name="time_of_flight" type="NX_INT" units="NX_TIME_OF_FLIGHT">
+  <doc>
+    NXevent_data is a special group for storing data from neutron
+    detectors in event mode.  In this mode, the detector electronics
+    emits a stream of detectorID, timestamp pairs. With detectorID
+    describing the detector element in which the neutron was detected
+    and timestamp the timestamp at which the neutron event was
+    detected. In NeXus detectorID maps to event_id, event_time_offset
+    to the timestamp.
+
+    As this kind of data is common at pulsed neutron
+    sources, the timestamp is almost always relative to the start of a
+    neutron pulse. Thus the pulse timestamp is recorded too together
+    with an index in the event_id, event_time_offset pair at which data for
+    that pulse starts. At reactor source the same pulsed data effect
+    may be achieved through the use of choppers or in stroboscopic
+    measurement setups. 
+
+    In order to make random access to timestamped data
+    faster there is an optional array pair of
+    cue_timestamp_zero and cue_index. The cue_timestamp_zero will
+    contain courser timestamps then in the time array, say
+    every five minutes. The cue_index will then contain the
+    index into the event_id,event_time_offset pair of arrays for that
+    courser cue_timestamp_zero. 
+
+
+
+  </doc>
+	<field name="event_time_offset" type="NX_INT" units="NX_TIME_OF_FLIGHT">
 		<doc>
-			A list of time of flight for each event as it comes in. 
-			This list is for all pulses with information to attach 
-			to a particular pulse located in events_per_pulse.
+			A list of timestamps for each event as it comes in. 
 		</doc>
 		<dimensions rank="1"><dim index="1" value="i"/></dimensions>
 	</field>
-	<field name="pixel_number" type="NX_INT" units="NX_DIMENSIONLESS">
+	<field name="event_id" type="NX_INT" units="NX_DIMENSIONLESS">
 		<doc>
 			There will be extra information in the NXdetector to convert 
-			pixel_number to detector_number. This list is for all pulses with 
-			information to attach to a particular pulse located in events_per_pulse.
+			event_id to detector_number.
 		</doc>
 		<dimensions rank="1"><dim index="1" value="i"/></dimensions>
 	</field>
-	<field name="pulse_time" type="NX_INT" units="NX_TIME">
+	<field name="event_time_zero" type="NX_INT" units="NX_TIME">
 		<doc>
 			The time that each pulse started with respect to the offset
 		</doc>
@@ -53,12 +77,11 @@
 			<doc>ISO8601</doc>
 		</attribute>
 	</field>
-	<field name="events_per_pulse" type="NX_INT" units="NX_DIMENSIONLESS">
-		<doc>
-			This connects the index "i" to the index "j". 
-			The jth element is the number of events in "i" 
-			that occurred during the jth pulse.
-		</doc>
+	<field name="event_index" type="NX_INT" units="NX_DIMENSIONLESS">
+	  <doc>
+	    The index into the event_time_offset, event_id pair for
+	    the pulse occurring at the matching entry in event_time_zero.
+	  </doc>
 		<dimensions rank="1"><dim index="1" value="j"/></dimensions>
 	</field>
 	<field name="pulse_height" type="NX_FLOAT" units="NX_DIMENSIONLESS">
@@ -73,5 +96,19 @@
 			<dim index="2" value="k"/>
 		</dimensions>
 	</field>
+	<field name="cue_timestamp_zero"  type="NX_DATE_TIME"
+	       units="NX_TIME">
+	  <doc>
+	    Timestamps matching the corresponding cue_index into the
+	    event_id, event_time_offset pair.
+	  </doc>
+	   <attribute name="start" type="NX_DATE_TIME" />
+	</field>
+	<field name="cue_index" type="NX_INT">
+	  <doc>
+	    Index into the event_id, event_time_offset pair matching the corresponding
+	    cue_timestamp. 
+	  </doc>
+	</field>
 </definition>
 

base_classes/NXlog.nxdl.xml

@@ -23,7 +23,7 @@
 -->
 <definition
 	name="NXlog" 
-	version="1.0" 
+	version="1.1" 
     type="group" 
     extends="NXobject"
 	category="base"
@@ -34,28 +34,60 @@
     <doc>
 		Information recorded as a function of time.
 
-		Description of information that is recorded against time, 
-		such as information monitored during the run. 
-		It contains
-		the logged values and the times at which they were measured as elapsed time since a starting
-		time recorded in ISO8601 format. This method of storing logged data helps to distinguish
+		Description of information that is recorded against
+		time. There are two common use cases for this:
+
+		- When logging data such as temperature during a run
+		- When data is taken in streaming mode data acquisition,
+		  i.e. just timestamp, value pairs are stored and
+		  correlated later in data reduction with other data, 
+
+
+		It both cases NXlog contains 
+		the logged or streamed  values and the times at which they were measured as elapsed time since a starting
+		time recorded in ISO8601 format. The time units are
+		specified in the units attribute. An optional scaling attribute
+		can be used to accomodate non standard clocks. 
+
+
+		This method of storing logged data helps to distinguish
 		instances in which a variable is a dimension scale of the data, in which case it is stored
 		in an :ref:`NXdata` group, and instances in which it is logged during the 
-		run, when it should be stored in an :ref:`NXlog` group.
+		run, when it should be stored in an :ref:`NXlog`
+		group.
+
+
 		Note: When using multiple :ref:`NXlog` groups, it is suggested to place 
 		them inside a :ref:`NXcollection` group.  In such cases, when
 		:ref:`NXlog` is used in another class,
 		:ref:`NXcollection`/:ref:`NXlog` is then constructed.
+
+		In order to make random access to timestamped data faster there is an optional array pair of
+		cue_timestamp_zero and cue_index. The cue_timestamp_zero will
+		contain courser timestamps then in the time array, say
+		every five minutes. The cue_index will then contain the
+		index into the time,value pair of arrays for that
+		courser cue_timestamp_zero. 
+
     </doc>
 	<field name="time" type="NX_FLOAT" units="NX_TIME">
 		<doc>
 			Time of logged entry. The times are relative to the "start" attribute 
-			and in the units specified in the "units" attribute.
+			and in the units specified in the "units"
+			attribute. Please note that absolute
+			timestamps under unix are relative to 1.1.1970:00:00.
         </doc>
         <attribute name="start" type="NX_DATE_TIME" />
+	<attribute name="scaling" type="NX_NUMBER"/>
 	</field>
 	<field name="value" units="NX_ANY" type="NX_NUMBER">
-		<doc>Array of logged value, such as temperature</doc>
+	  <doc>
+	    Array of logged value, such as temperature. If this is
+	    a single value the dimensionality is
+	    nEntries. However, NXlog can also be used to store
+	    multi dimensional time stamped data such as images. In
+	    this example the dimensionality of values would be value[nEntries,xdim,ydim].   
+	  </doc>
 	</field>
 	<field name="raw_value" units="NX_ANY" type="NX_NUMBER">
 		<doc>Array of raw information, such as thermocouple voltage</doc>
@@ -72,4 +104,18 @@
 	<field name="duration" type="NX_FLOAT" units="NX_ANY">
 		<doc>Total time log was taken</doc>
 	</field>
+	<field name="cue_timestamp_zero"  type="NX_DATE_TIME"
+	       units="NX_TIME">
+	  <doc>
+	    Timestamps matching the corresponding cue_index into the
+	    time, value pair.
+	  </doc>
+	  <attribute name="start" type="NX_DATE_TIME" />
+	</field>
+	<field name="cue_index" type="NX_INT">
+	  <doc>
+	    Index into the time, value pair matching the corresponding
+	    cue_timestamp. 
+	  </doc>
+	</field>
 </definition>