Draft for discussion: Surface lines to model road shapes and heights

.github/workflows/protobuf.yml

@@ -6,8 +6,18 @@ on:
     branches: [ master ]
 
 jobs:
+<<<<<<< HEAD
+<<<<<<< HEAD
   build-proto2-linux64:
     name: Build Proto2 Linux 64
+=======
+  build-cpp-linux64:
+    name: Build C++ Linux 64
+>>>>>>> Initial GitHub Actions CPP ProtoBuf Build
+=======
+  build-proto2-linux64:
+    name: Build Proto2 Linux 64
+>>>>>>> Add Python ProtoBuf build
 
     runs-on: ubuntu-18.04
 
@@ -17,6 +27,10 @@ jobs:
       with:
         submodules: true
 
+<<<<<<< HEAD
+<<<<<<< HEAD
+=======
+>>>>>>> Add Python ProtoBuf build
     - name: Setup Python
       uses: actions/setup-python@v2
       with:
@@ -25,9 +39,12 @@ jobs:
     - name: Install Python Dependencies
       run: python -m pip install --upgrade pip setuptools wheel pyyaml
 
+<<<<<<< HEAD
     - name: Install Doxygen
       run: sudo apt-get install doxygen graphviz
 
+=======
+>>>>>>> Add Python ProtoBuf build
     - name: Cache Dependencies
       id: cache-depends
       uses: actions/cache@v2
@@ -54,9 +71,14 @@ jobs:
     - name: Prepare C++ Build
       run: mkdir build
 
-    - name: Prepare Documentation Bukd
+    # Versioning
+    - name: Get versioning
+      id: get_version
+      run: echo ::set-output name=VERSION::$(git describe --always)
+
+    - name: Prepare Documentation Build
       run: |
-        sed -i 's/@VERSION_PATCH@/@VERSION_PATCH@_GitHub_MasterBranch/g' doxygen_config.cmake.in
+        sed -i 's/PROJECT_NUMBER\s*= @VERSION_MAJOR@.@VERSION_MINOR@.@VERSION_PATCH@/PROJECT_NUMBER = master (${{ steps.get_version.outputs.VERSION }})/g' doxygen_config.cmake.in
         echo "EXCLUDE_PATTERNS = */osi3/* */protobuf-3.15.8/* */proto2cpp/*" >> doxygen_config.cmake.in
         echo "GENERATE_TREEVIEW = YES" >> doxygen_config.cmake.in
 
@@ -111,15 +133,22 @@ jobs:
     - name: Install Python Dependencies
       run: python -m pip install --upgrade pip setuptools wheel pyyaml
 
+=======
+>>>>>>> Initial GitHub Actions CPP ProtoBuf Build
     - name: Cache Dependencies
       id: cache-depends
       uses: actions/cache@v2
       with:
+<<<<<<< HEAD
         path: protobuf-3.15.8
+=======
+        path: protobuf-3.11.3
+>>>>>>> Initial GitHub Actions CPP ProtoBuf Build
         key: ${{ runner.os }}-v1-depends
 
     - name: Download ProtoBuf
       if: steps.cache-depends.outputs.cache-hit != 'true'
+<<<<<<< HEAD
       run: curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protobuf-all-3.15.8.tar.gz && tar xzvf protobuf-all-3.15.8.tar.gz
 
     - name: Build ProtoBuf
@@ -155,3 +184,39 @@ jobs:
 
     - name: Run Python Tests
       run: python -m unittest discover tests
+=======
+      run: curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.11.3/protobuf-all-3.11.3.tar.gz && tar xzvf protobuf-all-3.11.3.tar.gz
+
+    - name: Build ProtoBuf
+      if: steps.cache-depends.outputs.cache-hit != 'true'
+      working-directory: protobuf-3.11.3
+      run: ./configure DIST_LANG=cpp --prefix=/usr && make
+
+    - name: Install ProtoBuf
+      working-directory: protobuf-3.11.3
+      run: sudo make install && sudo ldconfig
+
+    - name: Prepare C++ Build
+      run: mkdir build
+
+    - name: Configure C++ Build
+      working-directory: build
+      run: cmake ..
+
+    - name: Build C++
+      working-directory: build
+      run: cmake --build . --config Release -j 4
+<<<<<<< HEAD
+>>>>>>> Initial GitHub Actions CPP ProtoBuf Build
+=======
+
+    - name: Build Python
+      run: python setup.py build && python setup.py sdist 
+
+    - name: Install Python
+      run: python -m pip install .
+
+    - name: Run Python Tests
+      run: python -m unittest discover tests
+
+>>>>>>> Add Python ProtoBuf build

.gitignore

@@ -4,6 +4,7 @@ dist/
 doc/html
 doc/latex
 osi/
+osi3/
 
 # Specific extensions
 *.egg-info
@@ -20,6 +21,7 @@ cmake_install.cmake
 install_manifest.txt
 osi_version.proto
 version.py
+pyproject.toml
 
 # Eclipse-specific files, if any
 *.cproject

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "flatbuffers"]
+	path = flatbuffers
+	url = https://github.com/google/flatbuffers.git

CMakeLists.txt

@@ -5,6 +5,9 @@ project(open_simulation_interface)
 # set default compiler
 set(CMAKE_CXX_STANDARD 11)
 
+# Optional Flatbuffer support
+set(BUILD_FLATBUFFER OFF CACHE BOOLEAN "Build flatbuffer versions of libraries")
+
 # Set a default build type if none was specified
 set(default_build_type "Release")
 if(EXISTS "${CMAKE_SOURCE_DIR}/.git")
@@ -84,6 +87,48 @@ set(OSI_PROTO_FILES
 )
 
 protobuf_generate_cpp(PROTO_SRCS PROTO_HEADERS ${OSI_PROTO_FILES})
+set(FLAT_HEADERS "")
+if(BUILD_FLATBUFFER)
+  set(FLAT_FBS "")
+  add_subdirectory("flatbuffers"
+                   ${CMAKE_CURRENT_BINARY_DIR}/flatbuffers-build
+                   EXCLUDE_FROM_ALL)
+  file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/descriptor.fbs" "namespace osi3;")
+  file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/include")
+  list(APPEND FLAT_FBS "${CMAKE_CURRENT_BINARY_DIR}/descriptor.fbs")
+  foreach (proto ${OSI_PROTO_FILES})
+    get_filename_component(proto_base ${proto} NAME_WE)
+    set(fbs "${proto_base}.fbs")
+    add_custom_command(
+      OUTPUT "${fbs}"
+      COMMAND $<TARGET_FILE:flatc> -I "${PROTOBUF_IMPORT_DIRS}" -o "${CMAKE_CURRENT_BINARY_DIR}" --proto "${CMAKE_CURRENT_SOURCE_DIR}/${proto}"
+      DEPENDS "${proto}" flatc
+      WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+      COMMENT "Convert ${proto} to ${fbs} using flatc"
+    )
+    list(APPEND FLAT_FBS "${CMAKE_CURRENT_BINARY_DIR}/${fbs}")
+  endforeach()
+
+  foreach (flat ${FLAT_FBS})
+    get_filename_component(flat_base ${flat} NAME_WE)
+    set(fbs "${flat_base}.fbs")
+    set(fbh "${flat_base}_generated.h")
+    add_custom_command(
+      OUTPUT "include/${fbh}"
+      COMMAND $<TARGET_FILE:flatc> -o "${CMAKE_CURRENT_BINARY_DIR}/include" --cpp --gen-mutable --gen-name-strings --scoped-enums "${fbs}"
+      DEPENDS "${FLAT_FBS}" flatc
+      WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+      COMMENT "Process ${fbs} to ${fbh} using flatc"
+    )
+    list(APPEND FLAT_HEADERS "${CMAKE_CURRENT_BINARY_DIR}/include/${fbh}")
+  endforeach()
+
+  add_custom_target(${PROJECT_NAME}_fbs_build ALL DEPENDS "${FLAT_HEADERS}")
+  add_library(${PROJECT_NAME}_fbs INTERFACE)
+  target_include_directories(${PROJECT_NAME}_fbs INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/include/>)
+  target_include_directories(${PROJECT_NAME}_fbs SYSTEM INTERFACE $<INSTALL_INTERFACE:$<INSTALL_PREFIX>/include>)
+  target_link_libraries(${PROJECT_NAME}_fbs INTERFACE flatbuffers)
+endif()
 
 add_library(${PROJECT_NAME}_static STATIC ${PROTO_SRCS} ${PROTO_HEADERS})
 target_include_directories(${PROJECT_NAME}_static
@@ -168,7 +213,7 @@ install(FILES
         COMPONENT dev)
 
 # Header files
-install(FILES ${PROTO_HEADERS}
+install(FILES ${PROTO_HEADERS} ${FLAT_HEADERS}
         DESTINATION "${INSTALL_INCLUDE_DIR}")
 
 # Install the export set for use with the install-tree

README.md

@@ -30,7 +30,7 @@ def main():
 
     sv_ground_truth = sensorview.global_ground_truth
     sv_ground_truth.version.version_major = 3
-    sv_ground_truth.version.version_minor = 3
+    sv_ground_truth.version.version_minor = 4
     sv_ground_truth.version.version_patch = 0
 
     sv_ground_truth.timestamp.seconds = 0
@@ -91,7 +91,7 @@ Install `pip3` and missing python packages:
 ```bash
 $ sudo apt-get install python3-pip python3-setuptools
 ```
-Install `protobuf` 3.0.0:
+Install `protobuf`:
 ```bash
 $ sudo apt-get install libprotobuf-dev protobuf-compiler
 ```

VERSION

@@ -1,3 +1,3 @@
 VERSION_MAJOR = 3
-VERSION_MINOR = 3
-VERSION_PATCH = 1
+VERSION_MINOR = 4
+VERSION_PATCH = 0

doc/architecture/architecture_overview.adoc

@@ -0,0 +1,41 @@
+= Overview of OSI architecture
+
+OSI contains an object-based environment description that uses the message format of the https://github.com/protocolbuffers/protobuf/wiki[Protocol Buffer] library.
+Google developed and maintains the Protocol Buffer library.
+OSI defines top-level messages that are used to exchange data between separate models.
+Top-level messages define the `GroundTruth` interface, the `SensorData` interface, and – since OSI version 3.0.0 – the interfaces `SensorView`, `SensorViewConfiguration`, and `FeatureData`.
+
+The following figure shows the interfaces and models involved in modeling a sensor.
+
+.Open Simulation Interface overview
+image::{images_open_simulation_interface}/osi-context.png[1100]
+
+
+OSI also defines interfaces for traffic participant models.
+The `TrafficCommand` interface makes it possible to send commands to traffic participant models.
+The `TrafficUpdate` interface makes it possible to receive the updated state from traffic participant models.
+The following figure shows the interfaces of a generic traffic participant.
+
+.Interface of a traffic participant
+image::{images_open_simulation_interface}/osi-traffic-participant-principle.png[1100]
+
+Traffic participant models may use other OSI interfaces internally, for example, to model autonomous vehicles.
+The following figure shows a more advanced use case for traffic participants.
+
+.Traffic participant with sensor models, AD function, and dynamic model
+image::{images_open_simulation_interface}/osi-traffic-participant-advanced.png[1100]
+
+The `HostVehicleData` interface describes the measured internal states of a traffic participant.
+OSI currently provides only limited support for data structures that describe measured internal states of traffic participants.
+Actuator intentions are currently not covered by OSI and must be handled using a different data description format.
+
+All fields in an interface are set to `optional`.
+`required` is not used.
+This has been done to allow backward-compatible changes in the field.
+Additionally, this is the default behavior in Protocol Buffer version 3 that no longer has the `required` type.
+Setting all fields to `optional` thus ensures update compatibility.
+However, this does not mean that it is optional to fill the field.
+For the purpose of providing a complete interface, all existing fields should be set, unless not setting a field carries a specific meaning, as indicated in the accompanying comment.
+
+All field numbers equal to or greater than 10000 are available for user-specific extensions via custom fields.
+No future evolution of OSI will therefore use field numbers equal to or greater than 10000.

doc/architecture/data_layer.adoc

@@ -0,0 +1,14 @@
+= Data layer
+
+The OSI data layer is defined in the message specifications using the ProtoBuf IDL cite:[protobuf].
+This defines the data that can be transmitted using OSI, including the structure and the semantics of the data.
+
+Additionally, it specifies the encoding to be used when OSI data is transmitted.
+Currently, ProtoBuf encoding is used, but other encodings are possible with the ProtoBuf IDL.
+FlatBuffer encoding has been implemented as an experimental feature.
+
+The data layer does not directly define components and transmission routes.
+These are defined in the OSI packaging layer.
+There may be different packaging layer implementations using the shared data layer definitions.
+The data that is exchanged remains compatible regardless of the packaging layer implementation.
+The use of a shared data layer ensures easy bridging between different packaging layer implementations.

doc/architecture/environmental_effect_model.adoc

@@ -0,0 +1,9 @@
+= Environmental effect model
+
+Environmental effect models consume `SensorView` messages and produce `SensorView` messages.
+Environmental effect models may, for example, alter `SensorView` messages to include effects and phenomena caused by:
+
+* Shadows and occlusions
+* Weather effects
+* Physics of a sensor
+* Pre-processing of raw sensor data

doc/architecture/feature_data.adoc

@@ -0,0 +1,5 @@
+= Feature data
+
+`FeatureData` messages contain detected features in the reference frame of a sensor.
+`FeatureData` messages are generated from `GroundTruth` messages.
+They serve, for example, as an input to sensor models simulating object detection or feature fusion models.

doc/architecture/formatting_scripts.adoc

@@ -0,0 +1,52 @@
+= Trace-file formatting scripts
+
+The OSI repository contains Python scripts for converting trace files from one format to another.
+The formatting scripts are stored in `open-simulation-interface/format/`
+
+**txt2osi.py**
+
+`txt2osi.py` converts plain-text trace files to binary `.osi` trace files.
+This script takes the following parameters:
+
+`--data`, `-d`::
+String containing the path to the file with serialized data.
+
+`--type`, `-t`::
+Optional string describing the message type used to serialize data.
+`'SensorView'`, `'GroundTruth'`, or `'SensorData'` are permitted values.
+The default value is `'SensorView'`.
+
+`--output`, `-o`::
+Optional string containing the name of the output file.
+The default value is `'converted.osi'`.
+
+`--compress`, `-c`::
+Optional Boolean controlling whether to compress the output to an lzma file.
+`True`, or `False` are permitted values.
+The default value is `False`.
+
+**osi2read.py**
+
+`osi2read.py` converts trace files to human-readable `.txth` trace files.
+This script takes the following parameters:
+
+`--data`, `-d`::
+String containing the path to the file with serialized data.
+
+`--type`, `-t`::
+Optional string describing the message type used to serialize data.
+`'SensorView'`, `'GroundTruth'`, or `'SensorData'` are permitted values.
+The default value is `'SensorView'`.
+
+`--output`, `-o`::
+Optional string containing the name of the output file.
+The default value is `'converted.txth'`.
+
+`--format`, `-f`::
+Optional string containing the format type of the trace file.
+`'separated'`, or `None` are permitted values.
+The default value is `None`.
+
+**Related topics**
+
+* <<_osi_trace_file_formats>>

doc/architecture/ground_truth.adoc

@@ -0,0 +1,5 @@
+= Ground truth
+
+`GroundTruth` messages describe the simulated environment containing all simulated objects in the global coordinate system at consecutive time instances.
+They are based on data available to the simulation environment.
+`GroundTruth` messages are typically contained in `Sensorview` messages.

doc/architecture/logical_model.adoc

@@ -0,0 +1,6 @@
+= Logical model
+
+Logical models consume `SensorData` messages and produce `SensorData` messages.
+
+An example of a logical model is a sensor-fusion model, which combines the output of multiple sensor models to produce data with less uncertainty.
+Another use case is the fault-injection model which, contrary to a sensor-fusion model, may be used to increase uncertainties.

doc/architecture/packaging_layer.adoc

@@ -0,0 +1,13 @@
+= Packaging layer
+
+The OSI packaging layer specifies how components that use the OSI data layer, for example, sensor models, are packaged for exchange.
+
+This specifies model types and their mandatory and optional OSI inputs, OSI outputs, and parameter interfaces.
+A model type may be, for example, a sensor model or a traffic participant model.
+The packaging layer also specifies component technology standards.
+This makes it possible to encapsulate model types in easily exchangeable component packages that can be used across platforms and implementations.
+
+Multiple packaging layer implementations are possible within the OSI framework.
+The shared data layer ensures easy bridging between the different implementations.
+The currently defined central packaging layer is the OSI Sensor Model Packaging (OSMP) specification.
+It is based on FMI 2.0 cite:[fmi2.0] and uses certain additional conventions to allow packaging of OSI using models as FMUs.