Add server capabilities

.gitignore

@@ -4,6 +4,9 @@
 tests/snapshots/output
 tests/snapshots/tmp
 
+# Prometheus
+node_exporter
+
 # Editors
 .vscode/
 .idea/

Dockerfile

@@ -1,19 +1,24 @@
 FROM ubuntu:22.04
 
-RUN apt-get update \
-    && apt-get -y install software-properties-common sudo g++ \
-    && apt-get -y install pip vim
+RUN apt-get update
+RUN uname -a
+RUN apt-get -y install linux-headers-5.15.0-37-generic
+RUN apt-get -y install software-properties-common sudo g++
+RUN apt-get -y install pip vim
+RUN apt-get -y install iproute2 iputils-ping ncat
+RUN apt-get -y install jq
 RUN DEBIAN_FRONTEND=noninteractive apt-get install -y tshark
 
 # create a user and add them to sudoers
-RUN useradd -ms /bin/bash ipal && echo "ipal ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/ipal
-USER ipal
+# RUN useradd -ms /bin/bash ipal && echo "ipal ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/ipal
+# USER ipal
 
 WORKDIR /home/ipal/ipal_transcriber/
-COPY --chown=ipal . .
+# COPY --chown=ipal . .
+COPY . .
 
 # Install transcriber
 RUN sudo pip install .
 RUN sudo pip install -r requirements-dev.txt
 
-CMD /bin/bash
+CMD "/bin/bash"

requirements.txt

@@ -1,2 +1,3 @@
 pyshark
 python-dateutil
+prometheus-client

setup.py

@@ -11,7 +11,11 @@
         "ipal-combine",
     ],
     package_data={"": ["*.ipal", "*.pcapng"]},
-    install_requires=["pyshark", "python-dateutil"],
+    install_requires=[
+        "pyshark",
+        "python-dateutil",
+        "prometheus-client"
+    ],
     tests_require=["pytest"],
     url="https://github.com/fkie-cad/ipal_transcriber",
     author="Konrad Wolsing",

start-transcriber-push-ipal.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -e
+set -x
+set -E
+
+INTERFACE="$1" ; shift
+SERVERIP="$1" ; shift
+SERVERPORT="$1" ; shift
+
+# start prometheus node exporter if present
+echo "Starting Node Exporter"
+if [ -f node_exporter ]; then
+    ./node_exporter &
+fi
+
+# start transcriber process on interface $INTERFACE
+echo "Starting Transcriber on interface $INTERFACE"
+ipal-transcriber --interface $INTERFACE --ipal.output $INTERFACE.ipal $@ &
+
+# start server push of produced data to given server
+run_push_ipal () {
+  echo "waiting until IDS-Server on $SERVERIP is available"
+  until tail -F $INTERFACE.ipal | ncat -w5 $SERVERIP $SERVERPORT
+  do
+    echo "IDS-Server not ready... retry in 1s"
+    sleep 1
+  done 
+}
+until [ "$(cat $INTERFACE.ipal)" ]
+do 
+  echo "waiting for first packet to be transcribed"
+  sleep 1
+done
+run_push_ipal &
+
+wait -n 
+exit $?

start-transcriber-push-state.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -e
+set -x
+set -E
+
+INTERFACE="$1" ; shift
+SERVERIP="$1" ; shift
+SERVERPORT="$1" ; shift
+
+# start prometheus node exporter if present
+echo "Starting Node Exporter"
+if [ -f node_exporter ]; then
+    ./node_exporter &
+fi
+
+mkfifo IPAL
+# start transcriber process on interface $INTERFACE
+echo "Starting Transcriber on interface $INTERFACE"
+ipal-transcriber --interface $INTERFACE --ipal.output IPAL $@ &
+
+ipal-state-extractor --log INFO --ipal.input IPAL --state.out $INTERFACE.state timeslice &
+
+# tail cannot handle two file watches https://github.com/nxadm/tail/issues/30
+# start server push of produced data to given server
+run_push_state () {
+  echo "waiting until IDS-Server on $SERVERIP is available"
+  if ping $SERVERIP -c 1 -w 20
+  then
+    sleep 3
+    tail -F $INTERFACE.state | ncat -w5 $SERVERIP $SERVERPORT
+  fi 
+}
+
+until [ "$(cat $INTERFACE.state)" ]
+do
+  echo "waiting for first state to be transcribed"
+  sleep 1
+done
+run_push_state
+
+wait -n 
+exit $?

transcriber/packet_processor.py

@@ -6,6 +6,9 @@
 from transcriber.rule_processor import RuleProcessor
 from transcribers.utils import IpalIdCounter, get_all_transcribers
 
+# Prometheus metrics provider
+from prometheus_client import Counter, Summary
+TRANSCRIPTION_TIME = Summary('transcribe_processing_seconds', 'Time spent processing a packet')
 
 class PacketProcessor:
 
@@ -28,6 +31,7 @@ def __init__(self):
         if settings.rules:
             self.rule_processor = RuleProcessor(settings.rules)
 
+    @TRANSCRIPTION_TIME.time()
     def process_packet(self, pkt):
 
         # 0th pipeline step: store timing information for eval output

transcriber/tools/prometheus.py

@@ -0,0 +1,6 @@
+from prometheus_client import start_http_server, Counter
+
+class PrometheusClient():
+    def __init__(self) -> None:
+        # 9100 is the node_exporter, if present
+        start_http_server(9101)

transcriber/transcriber.py

@@ -314,6 +314,13 @@ def main():
             capture.apply_on_packets(pkt_processor.process_packet)
 
         elif args.interface:
+            # Initialize Prometheus Logging, when listening on interface
+            from transcriber.tools.prometheus import PrometheusClient
+            try:
+                _prom = PrometheusClient()
+            except Exception as e:
+                settings.logger.warn("Prometheus Client could not be started.", e)
+
             settings.source = args.interface
             capture = pyshark.LiveCapture(
                 interface=args.interface,

transcribers/iec104.py

@@ -4,6 +4,10 @@
 from transcriber.messages import IpalMessage, Activity
 from transcribers.transcriber import Transcriber
 
+# Prometheus metrics provider
+from prometheus_client import Summary
+PACKET_PARSE = Summary('packet_parse_iec104', 'Time spent to parse an IEC104 packet')
+PACKET_MATCH = Summary('packet_match_iec104', 'Time spent to match an IEC104 packet')
 
 class IEC104Transcriber(Transcriber):
 
@@ -62,6 +66,7 @@ class IEC104Transcriber(Transcriber):
     def matches_protocol(self, pkt):
         return "IEC60870_104" in pkt
 
+    @PACKET_PARSE.time()
     def parse_packet(self, pkt):
 
         msgs = []
@@ -359,6 +364,7 @@ def parse_I_format(self, src, dest, timestamp, pkt, pkt_index):  # noqa: C901
 
         return [m]
 
+    @PACKET_MATCH.time()
     def match_response(self, requests, response):
 
         if response.type == "U":