diff --git a/documentation/docs/assets/img/minio-ui.png b/documentation/docs/assets/img/minio-ui.png new file mode 100644 index 00000000..e5750fb4 Binary files /dev/null and b/documentation/docs/assets/img/minio-ui.png differ diff --git a/documentation/docs/assets/img/mqtt-explorer-global-broker-connection-china.png b/documentation/docs/assets/img/mqtt-explorer-global-broker-connection-china.png new file mode 100644 index 00000000..cc22bdec Binary files /dev/null and b/documentation/docs/assets/img/mqtt-explorer-global-broker-connection-china.png differ diff --git a/documentation/docs/assets/img/mqtt-explorer-global-broker-topics-exercise2.png b/documentation/docs/assets/img/mqtt-explorer-global-broker-topics-exercise2.png new file mode 100644 index 00000000..f1a23f2f Binary files /dev/null and b/documentation/docs/assets/img/mqtt-explorer-global-broker-topics-exercise2.png differ diff --git a/documentation/docs/assets/img/mqtt-explorer-new-connection.png b/documentation/docs/assets/img/mqtt-explorer-new-connection.png new file mode 100644 index 00000000..d43fd11f Binary files /dev/null and b/documentation/docs/assets/img/mqtt-explorer-new-connection.png differ diff --git a/documentation/docs/assets/img/mqtt-explorer-topics.png b/documentation/docs/assets/img/mqtt-explorer-topics.png new file mode 100644 index 00000000..fe1d3aa8 Binary files /dev/null and b/documentation/docs/assets/img/mqtt-explorer-topics.png differ diff --git a/documentation/docs/assets/img/mqtt-explorer-wis2box-broker.png b/documentation/docs/assets/img/mqtt-explorer-wis2box-broker.png index 6a9756b2..ac688927 100644 Binary files a/documentation/docs/assets/img/mqtt-explorer-wis2box-broker.png and b/documentation/docs/assets/img/mqtt-explorer-wis2box-broker.png differ diff --git a/documentation/docs/assets/img/wis2box-api-collections-initial.png b/documentation/docs/assets/img/wis2box-api-collections-initial.png new file mode 100644 index 00000000..5708b814 Binary files /dev/null and b/documentation/docs/assets/img/wis2box-api-collections-initial.png differ diff --git a/documentation/docs/assets/img/wis2box-api-stations.png b/documentation/docs/assets/img/wis2box-api-stations.png new file mode 100644 index 00000000..725a80db Binary files /dev/null and b/documentation/docs/assets/img/wis2box-api-stations.png differ diff --git a/documentation/docs/assets/img/wis2box-grafana-ui.png b/documentation/docs/assets/img/wis2box-grafana-ui.png new file mode 100644 index 00000000..28e9e3d4 Binary files /dev/null and b/documentation/docs/assets/img/wis2box-grafana-ui.png differ diff --git a/documentation/docs/assets/img/wis2box-mini-buckets.png b/documentation/docs/assets/img/wis2box-mini-buckets.png deleted file mode 100644 index d2bbe113..00000000 Binary files a/documentation/docs/assets/img/wis2box-mini-buckets.png and /dev/null differ diff --git a/documentation/docs/assets/img/wis2box-minio-buckets.png b/documentation/docs/assets/img/wis2box-minio-buckets.png new file mode 100644 index 00000000..e042e8b2 Binary files /dev/null and b/documentation/docs/assets/img/wis2box-minio-buckets.png differ diff --git a/documentation/docs/assets/img/wis2box-ui-empty.png b/documentation/docs/assets/img/wis2box-ui-empty.png index 87f030e3..1e05f0b9 100644 Binary files a/documentation/docs/assets/img/wis2box-ui-empty.png and b/documentation/docs/assets/img/wis2box-ui-empty.png differ diff --git a/documentation/docs/assets/img/wis2box-ui-explore-stations.png b/documentation/docs/assets/img/wis2box-ui-explore-stations.png new file mode 100644 index 00000000..be111eba Binary files /dev/null and b/documentation/docs/assets/img/wis2box-ui-explore-stations.png differ diff --git a/documentation/docs/assets/img/wis2box-webapp-create-station-save.png b/documentation/docs/assets/img/wis2box-webapp-create-station-save.png new file mode 100644 index 00000000..f83123b8 Binary files /dev/null and b/documentation/docs/assets/img/wis2box-webapp-create-station-save.png differ diff --git a/documentation/docs/assets/img/wis2box-webapp-create-station-select-topic.png b/documentation/docs/assets/img/wis2box-webapp-create-station-select-topic.png new file mode 100644 index 00000000..a21f1d39 Binary files /dev/null and b/documentation/docs/assets/img/wis2box-webapp-create-station-select-topic.png differ diff --git a/documentation/docs/assets/img/wis2box-webapp-create-station-success.png b/documentation/docs/assets/img/wis2box-webapp-create-station-success.png new file mode 100644 index 00000000..caf28fe7 Binary files /dev/null and b/documentation/docs/assets/img/wis2box-webapp-create-station-success.png differ diff --git a/documentation/docs/assets/img/wis2box-webapp-import-station-from-oscar.png b/documentation/docs/assets/img/wis2box-webapp-import-station-from-oscar.png new file mode 100644 index 00000000..c9acef08 Binary files /dev/null and b/documentation/docs/assets/img/wis2box-webapp-import-station-from-oscar.png differ diff --git a/documentation/docs/assets/img/wis2box-webapp-select-stations.png b/documentation/docs/assets/img/wis2box-webapp-select-stations.png new file mode 100644 index 00000000..c36d524e Binary files /dev/null and b/documentation/docs/assets/img/wis2box-webapp-select-stations.png differ diff --git a/documentation/docs/assets/img/wis2box-webapp-stations-with-one-station.png b/documentation/docs/assets/img/wis2box-webapp-stations-with-one-station.png new file mode 100644 index 00000000..69178b03 Binary files /dev/null and b/documentation/docs/assets/img/wis2box-webapp-stations-with-one-station.png differ diff --git a/documentation/docs/assets/img/wis2box-webapp.png b/documentation/docs/assets/img/wis2box-webapp.png new file mode 100644 index 00000000..79ccabf4 Binary files /dev/null and b/documentation/docs/assets/img/wis2box-webapp.png differ diff --git a/documentation/docs/practical-sessions/accessing-your-student-vm.md b/documentation/docs/practical-sessions/accessing-your-student-vm.md index e5f62377..83aa1c99 100644 --- a/documentation/docs/practical-sessions/accessing-your-student-vm.md +++ b/documentation/docs/practical-sessions/accessing-your-student-vm.md @@ -4,10 +4,22 @@ title: Accessing your student VM # Accessing your student VM +!!! abstract + + In this session you will practice accessing your student VM using SSH and WinSCP. + ## Introduction As part of locally run wis2box training sessions, you can access your personal student VM on the local training network named "WIS2-training". +Your student VM has the following software pre-installed: + +- Ubuntu 22.0.4.3 LTS [ubuntu-22.04.3-live-server-amd64.iso](https://releases.ubuntu.com/jammy/ubuntu-22.04.3-live-server-amd64.iso) +- Python 3.10.12 +- Docker 24.0.6 +- Docker Compose 2.21.0 +- Text editors: vim, nano + !!! note If you want to run this training outside of a local training session, you can provide your own instance using any Cloud Provider: @@ -16,18 +28,25 @@ As part of locally run wis2box training sessions, you can access your personal s - AWS (Amazon Web Services)  ec2-instance `t3a.medium`  - Azure (Microsoft) Azure Virtual Machine `standard_b2s` - Select Ubuntu Server 20.0.4 LTS as OS and run the setup script available in [student-vm-setup.zip](https://training.wis2box.wis.wmo.int/student-vm-setup.zip) on your instance to ensure you have all required software. + Select Ubuntu Server 22.0.4 LTS as OS and run setup instructions as described at https://docs.wis2box.wis.wmo.int/en/latest/user/getting-started.html#software-dependencies + + The release archive for wis2box as used in this training can be downloaded as follows: - If you are using the student VM provided during local WIS2 training sessions, the required software will already be installed. + ```bash + wget https://github.com/wmo-im/wis2box/releases/download/1.0b5/wis2box-setup-1.0b5.zip + unzip wis2box-setup-1.0b5.zip + ``` + + You can always find the latest 'wis2box-setup' archive at [https://github.com/wmo-im/wis2box/releases](https://github.com/wmo-im/wis2box/releases). -!!! note + The exercise material as used in this training can be downloaded as follows: - The student-VMs provided during WIS2 local training sessions have the following command-line editors pre-installed: + ```bash + wget https://training.wis2box.wis.wmo.int/exercise-materials.zip + unzip exercise-materials.zip + ``` - - vi - - vim - - nano - - emacs + If you are using the student VM provided during local WIS2 training sessions, the required software will already be installed. ## Connect to your student VM on the local training network @@ -59,64 +78,49 @@ passwd: password updated successfully ## Verify software versions -To be able to practice conversion to BUFR, the student VM comes with ecCodes, synop2bufr and csv2bufr pre-installed: +To be able to practice data conversion to BUFR, the student VM comes with ecCodes, synop2bufr and csv2bufr pre-installed. -Check the ecCodes version via the `bufr_dump` command: +To be able to run wis2box, the student VM is provided with with Python, Docker and Docker Compose pre-installed. +Check Python version: ```bash -bufr_dump -V -``` -returns: -```console - -ecCodes Version 2.28.0 -``` - -Check synop2bufr version: -```bash -synop2bufr --version -``` -returns: -```console -synop2bufr, version 0.4.1 -``` - -Check csv2bufr version: -```bash -csv2bufr --version +python3 --version ``` returns: ```console -csv2bufr, version 0.6.3 +Python 3.10.12 ``` -To be able to run wis2box, the student VM also comes with Python Docker and Docker Compose pre-installed. - Check docker version: ```bash docker --version ``` returns: ```console -Docker version 20.10.17, build 100c701 +Docker version 24.0.6, build ed223bc ``` Check Docker Compose version: ```bash -docker-compose --version +docker compose version ``` returns: ```console -docker-compose version 1.29.0, build unknown +Docker Compose version v2.21.0 ``` -Check Python version: +To ensure your user can run Docker commands your user has been added to the `docker` group. + +To test that your user can run docker hello-world, run the following command: ```bash -python3 --version +docker run hello-world ``` + returns: ```console -Python 3.8.10 +Hello from Docker! +This message shows that your installation appears to be working correctly. +... ``` ## Inspect the exercise materials @@ -128,10 +132,12 @@ ls ~/ ``` returns: ```console -exercise-materials wis2box-1.0b4 +exercise-materials wis2box-1.0b5 ``` -To access the material on your local machine rather than from the command line, you can use SCP. Using WinSCP, you can create a new SCP connection to your VM as follows: +You can use WinSCP to connect to your instance and inspect the contents of your home directory and download or upload files between your VM and your local PC. + +Using WinSCP, you can create a new SCP connection to your VM as follows: winscp-student-vm-scp.png @@ -139,23 +145,6 @@ And you should be able to see the following content: winscp-student-vm-exercise-materials.png -## Exercise 1: Editing files on your Student VM - -Connect to your Student VM using WinSCP and browse into the directory: `~/exercise-materials/accessing-your-student-vm/` - -Right-click on the file `hello_world.txt` and select **Edit -> Internal editor**. Edit this file by adding a message of your own and save your changes. - -winscp_internal_editor.png - -From within your SSH client check the content of the file `~/exercise-materials/accessing-your-student-vm/hello_world.txt`: - -```bash -cat ~/exercise-materials/accessing-your-student-vm/hello_world.txt -``` -And confirm you see the changes you made in the file. - -During the exercises you will be asked to edit files. It is up to you if you prefer to edit files from the command line in your SSH client (using `vi`/`vim`/`nano`/`emacs`) or using WinSCP. - ## Conclusion !!! success "Congratulations!" diff --git a/documentation/docs/practical-sessions/configuring-station-metadata.md b/documentation/docs/practical-sessions/configuring-station-metadata.md index 437b95f1..bbd9c6be 100644 --- a/documentation/docs/practical-sessions/configuring-station-metadata.md +++ b/documentation/docs/practical-sessions/configuring-station-metadata.md @@ -4,96 +4,155 @@ title: Configuring station metadata # Configuring station metadata +!!! abstract + + In this session you will practice configuring station metadata for your dataset. + ## Introduction -wis2box uses a fixed station metadata list that is used as part of its runtime operation. +wis2box has a collection of station metadata that is used to publish data on WIS2. Only data for stations configured in the wis2box station list will be published on your wis2box broker. The **WIGOS Station Identifier (WSI)** is used as the unique reference of the station which produced a specific set of observation data. -## Preparation +## Create an authorization token for collections/stations -Login to your student VM using SSH. +To edit stations via the **wis2box-webapp** you will first to need create an authorization token. -Ensure wis2box is running: +Login to your student VM and ensure you are in the `wis2box-1.0b5` directory: ```bash -cd ~/wis2box-1.0b4 -python3 wis2box-ctl.py start -python3 wis2box-ctl.py status +cd ~/wis2box-1.0b5 ``` -## Update the wis2box station list - -Pre-select a few stations in your country that you would consider for data publishing on WIS2. If you want to ingest your own data sample later during the WIS2 training, make sure to add the stations corresponding to your data. +Then login into the **wis2box-management** container with the following command: -Edit the file `~/wis2box-data/metadata/station/station_list.csv`: +```bash +python3 wis2box-ctl.py login +``` -For each new station, add a row to the end of the file with the following values: +Within the wis2box-container your can create an authorization token for a specific endpoint using the command: `wis2box auth add-token --path `. -- `station_name`: the human readable name of the station -- `wigos_station_identifier`: the WSI issued for the station -- `traditional_station_identifier`: the traditional station identifier if a WSI does not exist -- `facility_type`: the station/platform type (use **Land (fixed)** for land stations) -- `latitude`: the latitude, in decimal degrees -- `longitude`: the longitude, in decimal degrees -- `elevation`: station elevation, in metres above sea level -- `territory_name`: the human readable country name -- `wmo_region`: the Roman numeral of your country based on WMO Regional Associations +For example, to use a random automatically generated token for the `collections/stations` endpoint: -!!! tip - Ensure that latitude and longitude values are correctly signed (for example, use the minus sign [`-`] for southern or western hemispheres. +```{.copy} +wis2box auth add-token --path collections/stations +``` -### Using data from OSCAR +The output will look like this: -It is recommended to use station information from the [WMO OSCAR/Surface](https://oscar.wmo.int/surface) system where available. +```{.copy} +Continue with token: 7ca20386a131f0de384e6ffa288eb1ae385364b3694e47e3b451598c82e899d1 [y/N]? y +Token successfully created +``` -The script `~/exercise-materials/create-station-list/oscar2wis2box.py` can be used to add stations to your station list if they are available in OSCAR/Surface. +Or, if you want to define your own token for the `collections/stations` endpoint, you can use the following example: -For example to add the stations with WIGOS-IDs=0-20000-0-78970, 0-20000-0-78969 and 0-20000-0-78962 to your `station_list.csv`, run the following commands: +```{.copy} +wis2box auth add-token --path collections/stations MyCatIsCalledJessie! +``` -```bash -python3 ~/exercise-materials/station-list/oscar2wis2box.py 0-20000-0-78970 >> ~/wis2box-data/metadata/station/station_list.csv -python3 ~/exercise-materials/station-list/oscar2wis2box.py 0-20000-0-78969 >> ~/wis2box-data/metadata/station/station_list.csv -python3 ~/exercise-materials/station-list/oscar2wis2box.py 0-20000-0-78962 >> ~/wis2box-data/metadata/station/station_list.csv +Output: + +```{.copy} +Continue with token: MyCatIsCalledJessie! [y/N]? y +Token successfully created ``` -### Review your station list +!!! note "Exercise 1: Create an authorization token for collections/stations" -Check the content of your station list from the command line as follows: + Please create an authorization token for the `collections/stations` endpoint using the instructions above. -```bash -cat ~/wis2box-data/metadata/station/station_list.csv -``` +## add station metadata using the wis2box-webapp -Or open the file in WinSCP. +The **wis2box-webapp** provides a graphical user interface to edit station metadata. -Keep adding lines to `station_list.csv` and ensure you have at least three stations defined. +Open the **wis2box-webapp** in your browser by navigating to `http:///wis2box-app`: -## Publishing station metadata +wis2box-webapp -Login in to the **wis2box-management** container: +And select stations: -```bash -cd ~/wis2box-1.0b4/ -python3 wis2box-ctl.py login -``` +wis2box-webapp-select-stations + +When you click add 'add new station' you are asked to provide the WIGOS-station-identifier for the station you want to add: + +wis2box-webapp-import-station-from-oscar + +When you click search the station data is retrieved from OSCAR/Surface, please note that this can take a few seconds. + +Review the data returned by OSCAR/Surface and add missing data where required. Select a topic for the station and provide your authorization token for the `collections/stations` endpoint and click 'save': -Run the following command to publish your station metadata: +wis2box-webapp-create-station-save + +wis2box-webapp-create-station-success + +Go back to the station list and you will see the station you added: + +wis2box-webapp-stations-with-one-station + +!!! note "Exercise 2: Add station metadata" + + Please add three or more stations to the wis2box station metadata collection of your wis2box. + + Please use stations from your country if possible, especially if you brought your own data. + + Otherwise, you can use the following WIGOS-station-identifiers for testing purposes: + + - 0-20000-0-91334 + - 0-20000-0-96323 (note missing station elevation in OSCAR) + - 0-20000-0-96749 (note missing station elevation in OSCAR) + +!!! note "Deriving elevation information" + +If your station elevation is missing, there are online services that allow the lookup of the elevation using open elevation data. One such example is the [Open Topo Data API](https://www.opentopodata.org). + +For example, to get the elevation of the BMKG auditorium one would query the Open Topo Data API as follows: ```bash -wis2box metadata station publish-collection + wget -q -O - "https://api.opentopodata.org/v1/aster30m?locations=-6.15558,106.84204" +{ + "results": [ + { + "dataset": "aster30m", + "elevation": 7.0, + "location": { + "lat": -6.15558, + "lng": 106.84204 + } + } + ], + "status": "OK" +} ``` -Ensure that your new station metadata was published to the API, by navigating to `http://.wis2.training/oapi/collections/stations/items`: +## Review your station metadata + +After saving your station metadata, you can review the content of your station metadata in the **wis2box-webapp**: + +You can verify the updated stations are available in the **wis2box-api**: + +wis2box-api-stations + +You can also visit the **wis2box-ui** at `http://` and select "EXPLORE" on your dataset and you will see the stations you added: + +wis2box-ui-explore-stations + +!!! note "Exercise 3: Review your station metadata" + + Verify the stations you added are associated to your dataset by visiting the **wis2box-api** and **wis2box-ui** endpoints for your host in your browser. + +You also have the option to view/update/delete the station in the **wis2box-webapp**. Note that you are required to provide your authorization token for the `collections/stations` endpoint to update/delete the station. -stationlist_from_oscar +!!! note "Exercise 4: Update/delete station metadata" -Click on your station metadata record and inspect the content, noting how it relates to the content of the `station_list.csv` you have updated. + Please update/delete the station metadata for one of the stations you added using the **wis2box-webapp**. ## Conclusion !!! success "Congratulations!" In this practical session, you learned how to: - - update station metadata - - publish station metadata + - create an authorization token for the `collections/stations` endpoint + - add station metadata to the wis2box + - review stations associated to datasets in the **wis2box-ui** + - update/delete station metadata using the **wis2box-webapp** diff --git a/documentation/docs/practical-sessions/connecting-to-mqtt.md b/documentation/docs/practical-sessions/connecting-to-mqtt.md index 1ebc0d0a..b89960f8 100644 --- a/documentation/docs/practical-sessions/connecting-to-mqtt.md +++ b/documentation/docs/practical-sessions/connecting-to-mqtt.md @@ -4,38 +4,81 @@ title: Connecting to WIS2 over MQTT # Connecting to WIS2 over MQTT +!!! abstract + + In this session you will practice connecting to the WIS2 Global Broker using MQTT Explorer. + ## Introduction WIS2 uses the MQTT protocol to advertise the availability of weather/climate/water data. The WIS2 Global Broker subscribes to all WIS2 Nodes in the network and republishes the messages it receives. The Global Cache subscribes to the Global Broker, downloads the data in the message and then republishes the message on the `cache` topic with a new URL. The Global Discovery Catalogue publishes discovery metadata from the Broker and provides a search API. -As part of the WIS2 Pilot Phase in 2023, the following Global Broker services are available: +As part of the WIS2 Pilot Phase in 2023, two Global Brokers are available (hosted by China Meteorological Administration and Météo-France). + +This is an example of the WIS2 notification message structure for a message received on the topic `origin/a/wis2/arg/sabm/data/core/weather/surface-based-observations/synop`: + +```json +{ + "id": "fa587559-b02e-40a2-9fd5-2c141c39b130", + "type": "Feature", + "version": "v04", + "geometry": { + "type": "Point", + "coordinates": [ + -56.625, + -64.24139, + 208 + ] + }, + "properties": { + "data_id": "wis2/arg/sabm/data/core/weather/surface-based-observations/synop/WIGOS_0-20000-0-89055_20230926T190000", + "datetime": "2023-09-26T19:00:00Z", + "pubtime": "2023-09-26T18:48:51Z", + "integrity": { + "method": "sha512", + "value": "5a239b5d2ba7a04bd3b2fa44a73a9fb98167d0d4424d7fd19c0a83c75a7715212e2b97b5db6581bb3e1895b92232614d4cc4841ee9164baf47183c8403668dbd" + }, + "wigos_station_identifier": "0-20000-0-89055" + }, + "links": [ + { + "rel": "canonical", + "type": "application/x-bufr", + "href": "http://w2b.smn.gov.ar/data/2023-09-26/wis/arg/sabm/data/core/weather/surface-based-observations/synop/WIGOS_0-20000-0-89055_20230926T190000.bufr4", + "length": 254 + }, + { + "rel": "via", + "type": "text/html", + "href": "https://oscar.wmo.int/surface/#/search/station/stationReportDetails/0-20000-0-89055" + } + ] +} +``` -### Météo-France +In this practical session you will learn how to use the MQTT Explorer tool to review the topics available on this Global Broker and be able to display WIS2 notification messages. -- host: globalbroker.meteo.fr -- port: 8883 -- username: everyone -- password: everyone +MQTT Explorer is a helpful tool to browse and review the topic structure for a given MQTT broker and visually interact with the MQTT protocol. There exist numerous other MQTT client and server software, depending on your requirements and technical environment. + +To work with MQTT programmatically (for example, in Python), you can use MQTT client libraries such as [paho-mqtt](https://pypi.org/project/paho-mqtt) to connect to an MQTT broker and process incoming messages. + +## Using MQTT Explorer to connect to the Global Broker -### CMA +One way to view messages published by this Global Broker is using the MQTT Explorer which can be downloaded from the [MQTT Explorer website](https://mqtt-explorer.com). + +Open MQTT Explorer and add a new connection to the Global Broker hosted by China Meteorological Administration using the following details: - host: gb.wis.cma.cn - port: 8883 - username: everyone - password: everyone -In this practical session you will learn how to use the MQTT Explorer tool to review the topics available on this Global Broker and be able to display WIS2 notification messages. - -## Using MQTT Explorer to connect to the Global Broker - -One way to view messages published by this Global Broker is using the MQTT Explorer which can be downloaded from the [MQTT Explorer website](https://mqtt-explorer.com). - -Open MQTT Explorer and add a new connection as follows: - -mqtt-explorer-global-broker-connection +mqtt-explorer-global-broker-connection Click on the 'ADVANCED' button and add the following topics to subscribe to: +- `origin/#` +- `cache/#` + mqtt-explorer-global-broker-advanced !!! note @@ -46,48 +89,95 @@ Click on the 'ADVANCED' button and add the following topics to subscribe to: Click 'BACK', then 'SAVE' to save your connection and subscription details. Then click 'CONNECT': -At this point, the following should appear in the MQTT Explorer session: +Messages should start appearing in your MQTT Explorer session as follows: mqtt-explorer-global-broker-topics -You are now ready to start exploring the WIS2 topics and message structure and answer the following questions: +You are now ready to start exploring the WIS2 topics and message structure. -!!! question "Review the WIS2 topic structure" - Use MQTT to browse topic structure under the `origin` and `cache` topics. - - How can we distinguish the originating country providing the data? +## Exercise 1: Review the WIS2 topic structure + +Use MQTT to browse topic structure under the `origin` and `cache` topics. + +!!! question - How many countries are sharing data? + How can we distinguish the originating country providing the data? + +??? note "Click to reveal answer" -!!! question "Zambia data from origin" - Find the latest message received in the following topic: + We can distinguish the originating country by looking at the fourth level of the topic structure. For example, the following topic: `origin/a/wis2/zmb/zambia_met_service/data/core/weather/surface-based-observations/synop` - You can view the content of the WIS2 message in the "Value" section on the right hand side. + tells us that the data was published by Zambia (zmb). + + The fifth level of the topic structure provides the id of the centre where the data originated from + (in this case, the Zambia Meteorological Service). + +!!! question + + How can we distinguish the data type? + +??? note "Click to reveal answer" - What is the timestamp in UTC for when this data was published? + We can distinguish the data type by looking at the ninth level of the topic structure. For example, the following topic: - What is the URL from which we can download the data in BUFR format? + `origin/a/wis2/zmb/zambia_met_service/data/core/weather/surface-based-observations/synop` + + tells us that the data type is surface-based observations (synop). -!!! question "Zambia data from cache" - Find the latest message received on the following topic: +## Exercise 2: Review the WIS2 message structure - `cache/a/wis2/zmb/zambia_met_service/data/core/weather/surface-based-observations/synop` +Disconnect from MQTT Explorer and update the 'Advanced' sections to change the subscription to the following: - What is the timestamp in UTC for when this data was published? +* `origin/a/wis2/+/+/data/core/weather/surface-based-observations/synop` +* `cache/a/wis2/+/+/data/core/weather/surface-based-observations/synop` - What is the URL we can use to download the data? What is the difference between this URL and the URL in the previous question? +mqtt-explorer-global-broker-topics-exercise2 !!! note - MQTT Explorer is a helpful tool to review the topic structure for a given MQTT broker and visually work with the MQTT protocol. There exist many MQTT client and server software. - - To work with MQTT programmatically (for example, in Python), you can use MQTT client libraries such as [paho-mqtt](https://pypi.org/project/paho-mqtt/) to connect to an MQTT broker and process incoming messages. + The `+` wildcard is used to subscribe to all countries (fourth level) and centres (fifth level) while the remaining topic structure is fixed to ensure we subscribe to subtopics `data/core/weather/surface-based-observations/synop`. + +Messages should start to appear again. + +You can view the content of the WIS2 message in the "Value" section on the right hand side. + +!!! question + + How can we identify the timestamp that the data was published? And how can we identify the timestamp that the data was collected? + +??? note "Click to reveal answer" + + The timestamp that the data was published is contained in the `properties` section of the message with a key of `pubtime`. + + The timestamp that the data was collected is contained in the `properties` section of the message with a key of `datetime`. + +!!! question + + How can we download the data from the URL provided in the message? + +??? note "Click to reveal answer" + + The URL is contained in the `links` section with `rel="canonical"` and defined by the `href` key. + + You can copy the URL and paste it into a web browser to download the data. + +## Exercise 3: cache vs origin topics + +The same message is published on both the `origin` and `cache` topics. Find a message that has been published on both topics and compare the content. + +!!! question + + What is the difference between the messages published on the `origin` and `cache` topics? + +??? note "Click to reveal answer" + + The message published on the `origin` topic contains a URL to the original data. The message published on the `cache` topic contains a URL to the data cached by the Global Cache. ## Conclusion !!! success "Congratulations!" - Congratulations! In this practical session, you learned: + In this practical session, you learned: - how to subscribe to WIS2 Global Broker services using MQTT Explorer - the WIS2 topic structure diff --git a/documentation/docs/practical-sessions/wis2box-introduction.md b/documentation/docs/practical-sessions/wis2box-introduction.md index b21b2ead..22bba6db 100644 --- a/documentation/docs/practical-sessions/wis2box-introduction.md +++ b/documentation/docs/practical-sessions/wis2box-introduction.md @@ -1,73 +1,181 @@ --- -title: Initializing the wis2box +title: Initializing wis2box --- -# Initializing the wis2box +# Initializing wis2box -## Introduction +!!! abstract "Learning outcomes" -In this session you will prepare the initial configuration of your wis2box and start the services. + By the end of this session you will be able to: -!!! note "wis2box installation and configuration" - The latest wis2box release has been pre-installed on your student VM using the release archive available on GitHub: + * Run the `wis2box-create-config.py` script to create the initial configuration + * Start wis2box and check the status of its components + * Access the **wis2box-webapp**, API, MinIO UI and Grafana dashboard in a browser + * Connect to the local **wis2box-broker** using MQTT Explorer - ```bash - wget https://github.com/wmo-im/wis2box/releases/download/1.0b5/wis2box-setup-1.0b5.zip - unzip wis2box-setup-1.0b5.zip - ``` - - You can always find the latest 'wis2box-setup' archive at [https://github.com/wmo-im/wis2box/releases](https://github.com/wmo-im/wis2box/releases). +!!! note - All the required steps for installation and configuration of the wis2box can be found in the [wis2box-documentation](https://docs.wis2box.wis.wmo.int/en/latest/) + The current training materials are using wis2box-1.0b5. + + See [accessing-your-student-vm](accessing-your-student-vm.md) for instructions on how to download and install the wis2box software stack if you are running this training outside of a local training session. ## Preparation -Login to your designated VM with your username and password. +Login to your designated VM with your username and password and ensure you are in the `wis2box-1.0b5` directory: -## wis2box start and status +```bash +cd ~/wis2box-1.0b5 +``` + +## `wis2box-create-config.py` + +The `wis2box-create-config.py` script can be used to create the initial configuration of your wis2box. + +It will ask you a set of question to help setup your configuration. + +You will be able to review and update the configuration files after the script has completed. + +Run the script as follows: + +```bash +python3 wis2box-create-config.py +``` + +We recommend you use the directory `wis2box-data` in your home directory to store your configuration and data. +Note that you need to define the full path to this directory. + +For example if your username is `mlimper`, the full path to the directory is `/home/mlimper/wis2box-data`: + +```{.copy} +mlimper@student-vm-mlimper:~/wis2box-1.0b5$ python3 wis2box-create-config.py +Please enter the directory on the host where wis2box-configuration-files are to be stored: +/home/mlimper/wis2box-data +Configuration-files will be stored in the following directory: + /home/mlimper/wis2box-data +Is this correct? (y/n/exit) +y +The directory /home/mlimper/wis2box-data has been created. +``` + +Next, you will be asked to enter the URL for your wis2box. This is the URL that will be used to access the wis2box web application, API and UI. + +Please use `http://` as the URL. Remember that your hostname is defined by your `username.wis2.training` + +```{.copy} +Please enter the URL of the wis2box: + For local testing the URL is http://localhost + To enable remote access, the URL should point to the public IP address or domain name of the server hosting the wis2box. +http://mlimper.wis2.training +The URL of the wis2box will be set to: + http://mlimper.wis2.training +Is this correct? (y/n/exit) +``` + +We recommend that you use the option of random password generation when prompted for `WIS2BOX_STORAGE_PASSWORD` and `WIS2BOX_BROKER_PASSWORD`. + +Next you will be asked for the 3-letter ISO code for your country and centre-id for your wis2box. The centre-id can be a string of your choosing for the purpose of this training. + +```{.copy} +Please enter your 3-letter ISO country code: +nld +Please enter the centre-id for your wis2box: +maaike_test +The country-code will be set to: + nld +The centre-id will be set to: + maaike_test +Is this correct? (y/n/exit) +``` + +Next you will answer a set of question to generate discovery metadata templates for your wis2box. The answers do not need to be correct for the purpose of this training. + +```{.copy} +******************************************************************************** +Creating initial configuration for surface and upper-air data. +******************************************************************************** +Please enter the email address of the wis2box administrator: +mlimper@wmo.int +The email address of the wis2box administrator will be set to: + mlimper@wmo.int +Is this correct? (y/n/exit) +n +Please enter the email address of the wis2box administrator: +me@gmail.com +The email address of the wis2box administrator will be set to: + me@gmail.com +Is this correct? (y/n/exit) +y +Please enter the name of your organization: +Maaike-TEST +Your organization name will be set to: + Maaike-TEST +Is this correct? (y/n/exit) +y +Getting bounding box for "nld". +bounding box: -68.6255319,11.825,7.2274985,53.744395. +Do you want to use this bounding box? (y/n/exit) +y +Created new metadata file: /home/mlimper/wis2box-data/metadata/discovery/metadata-synop.yml +Created new metadata file: /home/mlimper/wis2box-data/metadata/discovery/metadata-temp.yml +``` -Navigate to the directory containing the wis2box software stack: +We will review the discovery metadata templates in a later session. + +Once the scripts is completed check the contents of the `wis2box.env` file in your current directory: + +```bash +cat ~/wis2box-1.0b5/wis2box.env +``` + +Or check the content of the file via WinSCP. + +!!! note + + The `wis2box.env` file contains environment variables defining the configuration of your wis2box. For more information consult the [wis2box-documentation](https://docs.wis2box.wis.wmo.int/en/latest/reference/configuration.html) + +You can also check the contents of the `data-mappings.yml` file in your wis2box data directory: ```bash +cat ~/wis2box-data/data-mappings.yml +``` + +Or check the content of the file via WinSCP. + +!!! note + + The `data-mappings.yml` define the plugins used to transform your data. For more information see [data pipeline plugins in the wis2box-documentation](https://docs.wis2box.wis.wmo.int/en/latest/reference/running/data-pipeline-plugins.html) + + +## wis2box start and status + +Ensure you are in the directory containing the wis2box software stack: + +```{.copy} cd ~/wis2box-1.0b5 ``` Start wis2box with the following command: -```bash +```{.copy} python3 wis2box-ctl.py start ``` Inspect the status with the following command: -```bash +```{.copy} python3 wis2box-ctl.py status ``` Repeat this command until all services are up and running. -!!! question - What services are running? Which ports are used for each service? !!! note "wis2box and Docker" wis2box runs as a set of Docker containers managed by docker-compose. - The services are defined in the various `docker-compose*.yml` which can be found in the `~/wis2box-1.0b4/` directory. + The services are defined in the various `docker-compose*.yml` which can be found in the `~/wis2box-1.0b5/` directory. The Python script `wis2box-ctl.py` is used to run the underlying Docker Compose commands that control the wis2box services. -## wis2box webapp - -Open a web browser and visit the page `http:///wis2box-webapp`. - -This is the (new) wis2box web-application where you can ingest SYNOP and csv data and manage your station metadata. - -## wis2box UI - -Open a web browser and visit the page `http://`. - -The wis2box UI will display your configured datasets and allow you to view surface based weather observations published by your wis2box. - ## wis2box API Open a new tab and navigate to the page `http:///oapi`. @@ -76,39 +184,146 @@ Open a new tab and navigate to the page `http:///oapi`. This is the wis2box API (running via the **wis2box-api** container). -To view collections currently published to the API, click `View the collections in this service`. - !!! question + What collections are currently available? + +??? note "Click to reveal answer" + + To view collections currently available through the API, click `View the collections in this service`: + + wis2box-api-collections.png + + The following collections are currently available: + + - Discovery metadata + - Station metadata + - Data notifications + !!! question + How many data notifications have been published? -!!! question - How many stations are configured? + +??? note "Click to reveal answer" + + Click on "Data notifications", then click on `Browse through the items of "Data Notifications"`. + + You will note that the page says "No items" as no Data notifications have been published yet. + +## wis2box webapp + +Open a web browser and visit the page `http:///wis2box-webapp`: + +wis2box-webapp.png + +This is the (new) wis2box web application to enable you to interact with your wis2box: + +- ingest ASCII and CSV data +- update/review your station metadata +- monitor notifications published on your wis2box-broker + +We will use this web application in a later session. + +## wis2box UI + +Open a web browser and visit the page `http://`: + +wis2box-ui.png + +The wis2box UI will display your configured datasets. For the surface-weather-observations/synop dataset, you can `explore` the data that has been ingested. + +The UI is currently empty, as datasets have not yet been configured. ## wis2box-broker Open the MQTT Explorer on your computer and prepare a new connection to connect to your broker (running via the **wis2box-broker** container). -Use the following connection details: +Click `+` to add a new connection: + +mqtt-explorer-new-connection.png + +Click on the 'ADVANCED' button and add the following topics to subscribe to: + +- `$SYS/#` +- `origin/a/wis2/#` +- `wis2box/#` + +mqtt-explorer-topics.png + +!!! note + + The messages published under the `$SYS` topic are system messages published by the mosquitto service itself. + + The messages published under topics starting with `origin/a/wis2/#` are the WIS2 data notifications published by the wis2box-broker. + + The messages published under topics starting with `wis2box` are internal messages between the various components of the wis2box software stack. + +Use the following connection details, making sure to replace the value of `` with your hostname and `` with the value from your `wis2box.env` file: - **Protocol: mqtt://** - **Host: ``** - **Port: 1883** - **Username: wis2box** -- **Password: wis2box** -- under 'ADVANCED', subscribe to the topics `$SYS/#` and `origin/#` +- **Password: ``** + +!!! note + + Check you wis2box.env for the value of your WIS2BOX_BROKER_PASSWORD. Make sure to click "SAVE" to store your connection details. +Then click "CONNECT" to connect to your **wis2box-broker**. + mqtt-explorer-wis2box-broker.png -Once you are connected, you should see statistics being published by your broker on the `$SYS` topic. +Once you are connected, you should see statistics being published by your broker on the `$SYS/#`. + +Later during the training you will use the MQTT connection you saved to view notifications published by your wis2box-broker. + +## MinIO UI + +Open a web browser and visit the page `http://:9001`: + +minio-ui.png + +This is the MinIO UI (running via the **wis2box-storage** container). + +The username and password are defined in the `wis2box.env` file in your wis2box data directory by the environment variables `WIS2BOX_STORAGE_USERNAME` and `WIS2BOX_STORAGE_PASSWORD`. + +Use the command below to check the values of these environment variables from the command line in your SSH session: + +```{.copy} +cat ~/wis2box-1.0b5/wis2box.env +``` + +Or check the content of the file via WinSCP. + +Try to login to your MinIO UI. You will see that there 3 buckets already defined: + +- `wis2box-incoming`: used to receive incoming data +- `wis2box-public`: used to store data that is made available in the WIS2 notifications, the content of this bucket is proxied as `/data` on your `WIS2BOX_URL` via the nginx container +- `wis2box-archive`: used to archive data from `wis2box-incoming` on a daily basis + +minio-ui-buckets.png + +!!! note + + The **wis2box-storage** container will send a notification on the **wis2box-broker** when data is received. The **wis2box-management** container is subscribed to all messages on `wis2box/#` and will receive these notifications, triggering the data pipelines defined in your `data-mappings.yml`. + +## Grafana UI + +Open a web browser and visit the page `http://:3000`: + +wis2box-grafana-ui.png + +This is the Grafana UI, where you can view the wis2box workflow monitoring dashboard. You can also access the logs of the various containers in the wis2box software stack via the 'Explore' option in the menu. ## Conclusion !!! success "Congratulations!" In this practical session, you learned how to: - - Run the wis2box-create-config script to create the initial configuration + - Run the `wis2box-create-config.py` script to create the initial configuration - Start wis2box and check the status of its components - - access the wis2box-webapp, API, MinIO UI and Grafana dashboard + - Access the **wis2box-webapp**, API, MinIO UI and Grafana dashboard in a browser + - Connect to the **wis2box-broker** using MQTT Explorer diff --git a/environment/README.md b/environment/README.md index 72f7c5e7..739fa2f7 100644 --- a/environment/README.md +++ b/environment/README.md @@ -60,3 +60,9 @@ order to create MinIO storage. The following buckets must be created manually w - `exercise-materials` - `documentation` + +### student VM setup + +Student VMs can be setup by cloning the a base template. + +The script `setup_student_vm.sh` can be used to by the `wmo_admin` account to create a new user account on the student VM and add the latest wis2box release to the home directory of the new user along with the exercise materials. diff --git a/environment/preingest_data.sh b/environment/preingest_data.sh deleted file mode 100644 index def946ab..00000000 --- a/environment/preingest_data.sh +++ /dev/null @@ -1,18 +0,0 @@ -# pre-ingest data -sudo cp /tmp/dev.env wis2box-1.0b4 -cd wis2box-1.0b4 -python3 wis2box-ctl.py start -sleep 30 -python3 wis2box-ctl.py execute wis2box data add-collection /data/wis2box/metadata/discovery/mwi-surface-weather-observations.yml -python3 wis2box-ctl.py execute wis2box data add-collection /data/wis2box/metadata/discovery/mwi-surface-weather-observations-reco.yml -python3 wis2box-ctl.py execute wis2box metadata discovery publish /data/wis2box/metadata/discovery/mwi-surface-weather-observations.yml -python3 wis2box-ctl.py execute wis2box metadata discovery publish /data/wis2box/metadata/discovery/mwi-surface-weather-observations-reco.yml -python3 wis2box-ctl.py execute wis2box auth add-token --topic-hierarchy mwi.mwi_wmo_demo.data.recommended.weather.surface-based-observations.synop mysecrettoken -y -python3 wis2box-ctl.py execute wis2box auth add-token -p mwi_wmo_demo:reco mysecrettoken -y -python3 wis2box-ctl.py execute wis2box data ingest -th mwi.mwi_wmo_demo.data.core.weather.surface-based-observations.synop -p /data/wis2box/observations/malawi-preingested-core/ -python3 wis2box-ctl.py execute wis2box data ingest -th mwi.mwi_wmo_demo.data.recommended.weather.surface-based-observations.synop -p /data/wis2box/observations/malawi-preingested-reco/ -echo "sleep before refresh station links..." -sleep 120 -echo "publishing station metadata" -python3 wis2box-ctl.py execute wis2box metadata station publish-collection -echo "done!" diff --git a/environment/setup_new_user.sh b/environment/setup_new_user.sh deleted file mode 100644 index a20558fa..00000000 --- a/environment/setup_new_user.sh +++ /dev/null @@ -1,32 +0,0 @@ -# /bin/bash -# this is a simple script to create a new user on the local training hardware -# and ensure the user has required materials in the home-directory - -read -p "USERNAME: " USERNAME - -echo "USERNAME=`echo $USERNAME`" -read -p "Continue? (Y/N): " confirm && [[ $confirm == [yY] || $confirm == [yY][eE][sS] ]] || exit 1 - -sudo useradd -p $(openssl passwd -1 "wis2training") "$USERNAME" -m -G docker,sudo --shell /bin/bash -sudo cp -rf exercise-materials/ /home/`echo $USERNAME`/exercise-materials/ -sudo cp -rf wis2box-1.0b4/ /home/`echo $USERNAME`/wis2box-1.0b4/ - -echo WIS2BOX_HOST_DATADIR=/home/`echo $USERNAME`/exercise-materials/wis2box-test-data/ > /tmp/dev.env -echo WIS2BOX_URL=http://`echo $USERNAME`.wis2.training >> /tmp/dev.env -echo WIS2BOX_API_URL=http://`echo $USERNAME`.wis2.training/oapi >> /tmp/dev.env -echo WIS2BOX_LOGGING_LOGLEVEL=WARNING >> /tmp/dev.env -sudo cp /tmp/dev.env /home/`echo $USERNAME`/wis2box-1.0b4/ - -# create ftp.env -echo MYHOSTNAME=`echo $USERNAME`.wis2.training > /tmp/ftp.env -echo FTP_USER=wis2box >> /tmp/ftp.env -echo FTP_PASS=wis2box >> /tmp/ftp.env -echo FTP_HOST=`echo $USERNAME`.wis2.training >> /tmp/ftp.env -echo "WIS2BOX_STORAGE_ENDPOINT=http://`echo $USERNAME`.wis2.training:9000" >> /tmp/ftp.env -echo WIS2BOX_STORAGE_USERNAME=minio >> /tmp/ftp.env -echo WIS2BOX_STORAGE_PASSWORD=minio123 >> /tmp/ftp.env -echo LOGGING_LEVEL=WARNING >> /tmp/ftp.env -sudo cp /tmp/ftp.env /home/`echo $USERNAME`/wis2box-1.0b4/ - -sudo chown -R `echo $USERNAME`:`echo $USERNAME` /home/`echo $USERNAME` -sudo cat /home/`echo $USERNAME`/wis2box-1.0b4/dev.env \ No newline at end of file diff --git a/environment/setup_student_vm.sh b/environment/setup_student_vm.sh new file mode 100644 index 00000000..3992978e --- /dev/null +++ b/environment/setup_student_vm.sh @@ -0,0 +1,59 @@ +# /bin/bash +# this is a simple script to create a new user on the local training hardware +# and ensure the user has required materials in the home-directory + +# throw and error if no arguments are provided +if [ $# -eq 0 ] + then + echo "No arguments supplied" + echo "Usage: ./setup_student_vm.sh " + exit 1 +fi + +# first argument of the script is the username +USERNAME=$1 +# second argument is the IP of the student-vm +HOST_IP=$2 + +echo "USERNAME=`echo $USERNAME`" +read -p "Continue? (Y/N): " confirm && [[ $confirm == [yY] || $confirm == [yY][eE][sS] ]] || exit 1 +echo "HOST_IP=`echo $HOST_IP`" +read -p "Continue? (Y/N): " confirm && [[ $confirm == [yY] || $confirm == [yY][eE][sS] ]] || exit 1 + +# execute commands over ssh + +# create the user, if not already existing +ssh wmo_admin@`echo $HOST_IP` "sudo useradd "$USERNAME" -m -G docker --shell /bin/bash" +# set the initial password to "wis2training" +ssh wmo_admin@`echo $HOST_IP` "echo "$USERNAME":wis2training | sudo chpasswd" +# rename the hostname to student-vm- +ssh wmo_admin@`echo $HOST_IP` "sudo hostnamectl set-hostname student-vm-`echo $USERNAME`" + +# copy the latest wis2box-setup-1.0b5.zip to the student-vm +ssh wmo_admin@`echo $HOST_IP` "wget https://github.com/wmo-im/wis2box/releases/download/1.0b5/wis2box-setup-1.0b5.zip -O /tmp/wis2box-setup-1.0b5.zip" +# unzip the wis2box-setup-1.0b5.zip +ssh wmo_admin@`echo $HOST_IP` "sudo unzip -o /tmp/wis2box-setup-1.0b5.zip -d /home/`echo $USERNAME`/" +# remove the wis2box-setup-1.0b5.zip +ssh wmo_admin@`echo $HOST_IP` "rm -rf /tmp/wis2box-setup-1.0b5.zip" + +# copy the latest exercise materials to the student-vm +ssh wmo_admin@`echo $HOST_IP` "wget https://training.wis2box.wis.wmo.int/exercise-materials.zip -O /tmp/exercise-materials.zip" +# unzip the exercise-materials.zip +ssh wmo_admin@`echo $HOST_IP` "sudo unzip -o /tmp/exercise-materials.zip -d /home/`echo $USERNAME`/" +# remove the exercise-materials.zip +ssh wmo_admin@`echo $HOST_IP` "rm -rf /tmp/exercise-materials.zip" + +# create ftp.env +ssh wmo_admin@`echo $HOST_IP` "echo MYHOSTNAME=`echo $USERNAME`.wis2.training > /tmp/ftp.env" +ssh wmo_admin@`echo $HOST_IP` "echo FTP_USER=wis2box >> /tmp/ftp.env" +ssh wmo_admin@`echo $HOST_IP` "echo FTP_PASS=wis2box >> /tmp/ftp.env" +ssh wmo_admin@`echo $HOST_IP` "echo FTP_HOST=`echo $USERNAME`.wis2.training >> /tmp/ftp.env" +ssh wmo_admin@`echo $HOST_IP` "echo "WIS2BOX_STORAGE_ENDPOINT=http://`echo $USERNAME`.wis2.training:9000" >> /tmp/ftp.env" +ssh wmo_admin@`echo $HOST_IP` "echo WIS2BOX_STORAGE_USERNAME=minio >> /tmp/ftp.env" +ssh wmo_admin@`echo $HOST_IP` "echo WIS2BOX_STORAGE_PASSWORD=minio123 >> /tmp/ftp.env" +ssh wmo_admin@`echo $HOST_IP` "echo LOGGING_LEVEL=WARNING >> /tmp/ftp.env" +ssh wmo_admin@`echo $HOST_IP` "sudo cp /tmp/ftp.env /home/`echo $USERNAME`/wis2box-1.0b5/" +# remove /tmp/ftp.env +ssh wmo_admin@`echo $HOST_IP` "rm -rf /tmp/ftp.env" + +ssh wmo_admin@`echo $HOST_IP` "sudo chown -R `echo $USERNAME`:`echo $USERNAME` /home/`echo $USERNAME`" \ No newline at end of file diff --git a/environment/student-vm/student-vm-setup.sh b/environment/student-vm/student-vm-setup.sh deleted file mode 100644 index 5a7df9d4..00000000 --- a/environment/student-vm/student-vm-setup.sh +++ /dev/null @@ -1,64 +0,0 @@ -# this is the wis2box base setup script to create the student-vm as used in the wis2box-training -# using base OS: Ubuntu 20.04 LTS (Focal) - -# install ecCodes, pymetdecoder, csv2bufr, synop2bufr -ECCODES_VER=2.28.0 -ECCODES_DIR=/opt/eccodes -PATH="$PATH;/opt/eccodes/bin" -cd /tmp -sudo apt-get remove -y libeccodes-tools -sudo apt-get install -y build-essential cmake gfortran python3-dev curl unzip -curl https://confluence.ecmwf.int/download/attachments/45757960/eccodes-${ECCODES_VER}-Source.tar.gz --output eccodes-${ECCODES_VER}-Source.tar.gz -tar xzf eccodes-${ECCODES_VER}-Source.tar.gz -mkdir build -cd build -sudo cmake -DCMAKE_INSTALL_PREFIX=${ECCODES_DIR} -DENABLE_AEC=OFF ../eccodes-${ECCODES_VER}-Source -sudo make -sudo ctest -sudo make install -cd ~/ -sudo rm -rf /tmp/build -# update path in .bashrc -echo 'export PATH=$PATH:/opt/eccodes/bin' >> ~/.bashrc -echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/eccodes/lib' >> ~/.bashrc -# update current paths -export PATH="$PATH:/opt/eccodes/bin" -export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/eccodes/lib" - -sudo pip3 install https://github.com/wmo-im/pymetdecoder/archive/refs/tags/v0.1.7.zip -sudo pip3 install https://github.com/wmo-im/csv2bufr/archive/refs/tags/v0.6.3.zip -sudo pip3 install https://github.com/wmo-im/synop2bufr/archive/refs/tags/v0.4.1.zip -python3 -m eccodes selfcheck - -# install various utilities -sudo apt-get -y install mosquitto-clients nano vim - -# install MinIO, pywis-pubsub, pywiscat -sudo pip3 install minio==7.1.13 -sudo pip3 install pywis-pubsub -sudo pip3 install https://github.com/wmo-im/pywiscat/archive/master.zip -sudo pip3 install https://github.com/wmo-cop/pyoscar/archive/master.zip - -# install the Docker engine, Python 3.8 and Docker Compose -sudo mkdir -m 0755 -p /etc/apt/keyrings -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg -sudo chmod a+r /etc/apt/keyrings/docker.gpg -sudo echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null -sudo apt-get -y update -sudo apt-get install -y docker-ce python3.8 python3-pip -sudo pip3 install pip --upgrade -sudo pip3 install pyopenssl --upgrade -sudo pip3 install docker-compose==1.29.2 requests==2.26.0 urllib3==1.26.0 - -# (re-)download wis2box -cd ~/ -rm -rf wis2box-1.0b4/ -wget https://github.com/wmo-im/wis2box/releases/download/1.0b4/wis2box-setup-1.0b4.zip -unzip wis2box-setup-1.0b4.zip -rm wis2box-setup-1.0b4.zip - -# (re-)download exercise materials -rm -rf exercise-materials/ -wget https://training.wis2box.wis.wmo.int/exercise-materials.zip -unzip exercise-materials.zip -rm -rf exercise-materials.zip