Merge pull request #35 from BenjaminBruenau/develop

CI Improvements and better Report Generation

.github/workflows/release-dev.yaml

@@ -48,6 +48,14 @@ jobs:
         run: sbt publishLocal
         working-directory: ./backend/commons
 
+      - name: Build and Push Analysis Service
+        run: |
+          sbt assembly
+          docker build -t analysis:0.1.0 .
+          docker tag analysis:0.1.0 ${{ vars.GCLOUD_REGION }}-docker.pkg.dev/${{vars.PROJECT_ID}}/ruettel-report-dev/analysis:0.1.0
+          docker push ${{ vars.GCLOUD_REGION }}-docker.pkg.dev/${{vars.PROJECT_ID}}/ruettel-report-dev/analysis:0.1.0
+        working-directory: ./backend/Analysis
+
       - name: Build Images
         run: sbt docker:publishLocal
         working-directory: ${{ env.working-directory }}
@@ -60,7 +68,7 @@ jobs:
         name: Tag DataTransformerService Docker image and push to Google Artifact Registry
         run: |
           docker tag datatransformerservice:0.1.0 ${{ vars.GCLOUD_REGION }}-docker.pkg.dev/${{vars.PROJECT_ID}}/ruettel-report-dev/datatransformerservice:0.1.0
-          docker push ${{ vars.GCLOUD_REGION }}-docker.pkg.dev/${{vars.PROJECT_ID}}/ruettel-report-dev/queryservice:0.1.0
+          docker push ${{ vars.GCLOUD_REGION }}-docker.pkg.dev/${{vars.PROJECT_ID}}/ruettel-report-dev/datatransformerservice:0.1.0
       - id: docker-build-push-frontend-and-projectmanagement-service-tagged
         name: Build,Tag Frontend and ProjectManagement Docker image and push to Google Artifact Registry
         run: |

.github/workflows/release.yaml

@@ -49,6 +49,14 @@ jobs:
         run: sbt publishLocal
         working-directory: ./backend/commons
 
+      - name: Build and Push Analysis Service
+        run: |
+          sbt assembly
+          docker build -t analysis:0.1.0 .
+          docker tag analysis:0.1.0 ${{ vars.GCLOUD_REGION }}-docker.pkg.dev/${{vars.PROJECT_ID}}/ruettel-report/analysis:${{ steps.get-tag.outputs.short_ref }}
+          docker push ${{ vars.GCLOUD_REGION }}-docker.pkg.dev/${{vars.PROJECT_ID}}/ruettel-report/analysis:${{ steps.get-tag.outputs.short_ref }}
+        working-directory: ./backend/Analysis
+
       - name: Build Images
         run: sbt docker:publishLocal
         working-directory: ${{ env.working-directory }}

.helm/ruettel-chart-local-values.yaml

@@ -1,7 +1,7 @@
 
 
 image:
-  repository: europe-west6-docker.pkg.dev/instant-heading-405914/ruettel-report/
+  repository: europe-west6-docker.pkg.dev/instant-heading-405914/ruettel-report-dev/  # or europe-west6-docker.pkg.dev/instant-heading-405914/ruettel-report/
   pullPolicy: IfNotPresent  #Always
   #Overrides the image tag whose default is the chart appVersion.
   tag: "1.0.6"

README.md

@@ -7,6 +7,97 @@
 [![codecov](https://codecov.io/gh/BenjaminBruenau/RuettelReport/graph/badge.svg?token=7OKGD5WV2H)](https://codecov.io/gh/BenjaminBruenau/RuettelReport)
 
 
+## Local Setup Guide 
+**(with local Postgres DB for FusionAuth)**
+
+### Clone Project
+
+````shell
+git clone https://github.com/BenjaminBruenau/RuettelReport
+````
+
+### Start local Cluster
+
+We recommend to start the local Cluster with a good chunk of memory, as otherwise running both SparkApplications and 
+Kafka will result in Pods being killed (_OOMKilled_).
+```shell
+minikube start --cpus 5 --memory 8g
+```
+To use images from a private Artifact Registry (like our development and production Repositiories there) it is necessary to mount the K8s Pods with the required Google Cloud Credentials.
+```shell
+minikube addons enable gcp-auth
+```
+
+### Installing RuettelReport Infrastructure
+```shell
+helm install spark spark-operator/spark-operator --namespace spark-operator --create-namespace -f spark-operator-values.yaml
+
+helm install mongodb bitnami/mongodb -f mongodb-values.yaml -n shared --create-namespace
+
+helm install kong kong/ingress -n kong --create-namespace 
+
+helm install pg-minikube --set auth.postgresPassword=admin bitnami/postgresql
+helm install my-fusion fusionauth/fusionauth -f local-fa-values.yaml
+```
+#### With Monitoring (optional)
+
+```shell
+helm install promstack prometheus-community/kube-prometheus-stack --namespace monitoring --version 52.1.0 -f values-monitoring.yaml
+helm upgrade kong kong/ingress -n kong --set gateway.serviceMonitor.enabled=true --set gateway.serviceMonitor.labels.release=promstack
+kubectl apply -f kong-prometheus-plugin.yaml
+```
+
+### Installing the RuettelReport Application Chart
+
+**!The `ruettel-chart-local-values.yaml` needs to be adjusted before!**
+
+1. [Port Forward](#FusionAuth) FusionAuth
+2. Access its UI in the browser
+3. Go to `Settings` -> `Key Manager`
+4. View the `premium` and `free` key and copy both their public key entries
+5. Replace the values for `kong.premiumConsumerSecret` and `kong.freeConsumerSecret` (in `ruettel-chart-local-values.yaml`)
+with their corresponding public key value
+6. Proceed with the Application Chart Installation
+
+```shell
+helm install ruettel-chart ./ruettel-chart -f ruettel-chart-local-values.yaml --set image.tag=<your desired release version / latest>
+```
+
+### Accessing Infrastructure Services
+
+#### Monitoring
+
+````shell
+kubectl -n monitoring port-forward services/prometheus-operated 9090 & kubectl -n monitoring port-forward services/promstack-grafana 3000:80 &
+
+kubectl get secret --namespace monitoring promstack-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo
+````
+
+#### FusionAuth
+
+````shell
+export SVC_NAME=$(kubectl get svc --namespace default -l "app.kubernetes.io/name=fusionauth,app.kubernetes.io/instance=my-fusion" -o jsonpath="{.items[0].metadata.name}")
+kubectl port-forward svc/$SVC_NAME 9011:9011
+````
+
+#### MongoDB
+
+````shell
+kubectl port-forward svc/mongodb-headless 27017:27017
+````
+
+#### Spark Operator
+
+````shell
+kubectl describe sparkapplication spark-analysis -n premium
+````
+
+Get Logs of specific SparkApplication Job:
+````shell
+kubectl logs spark-analysis-driver -n premium
+````
+
+
 ## Architecture
 
 

frontend/assets/presets/primevue-preset.js

@@ -28,9 +28,11 @@ const PrimevueDesignPreset = {
         header: ({ props: PanelProps }) => ({
             class: [
                 'flex items-center justify-between', // flex and alignments
-                'border border-gray-300 bg-gray-100 text-gray-700 rounded-tl-lg rounded-tr-lg', // borders and colors
-                'dark:bg-gray-900 dark:border-blue-900/40 dark:text-white/80', // Dark mode
-                { 'p-5': !props.toggleable, 'py-3 px-5': props.toggleable } // condition
+
+                { 'p-5': !props.toggleable, 'py-3 px-5': props.toggleable }, // condition
+                'bg-tile_color_light dark:bg-tile_color_dark',
+                'border-b_color_light dark:border-b_color_dark',
+                'text-textColor_light dark:text-textColor_dark',
             ]
         }),
         title: {
@@ -49,8 +51,10 @@ const PrimevueDesignPreset = {
         },
         content: {
             class: [
-                'p-5 border border-gray-300 bg-white text-gray-700 border-t-0 last:rounded-br-lg last:rounded-bl-lg',
-                'dark:bg-gray-900 dark:border-blue-900/40 dark:text-white/80' // Dark mode
+                'p-5 border-t-0 last:rounded-br-lg last:rounded-bl-lg',
+                'bg-tile_color_light dark:bg-tile_color_dark',
+                'border-b_color_light dark:border-b_color_dark',
+                'text-textColor_light dark:text-textColor_dark',
             ] // padding, borders, and colors
         }
     },
@@ -279,6 +283,7 @@ const PrimevueDesignPreset = {
                 'bg-tile_color_light dark:bg-tile_color_dark border-border_color_light dark:border-border_color_dark',
                 'border-b_color_light dark:border-b_color_dark',
                 'text-textColor_light dark:text-textColor_dark',
+                'bg-gradient-to-r from-mainColor_1_1_light to-mainColor_1_2_light dark:from-mainColor_1_1_dark dark:to-mainColor_1_2_dark'
             ]
         }),
         container: ({ props }) => ({
@@ -323,6 +328,31 @@ const PrimevueDesignPreset = {
                 'text-textColor_light dark:text-textColor_dark',
             ]
         }),
+        previousbutton: {
+            class: ['text-textColor_light dark:text-textColor_dark']
+        },
+        monthTitle: {
+            class: ['text-textColor_light dark:text-textColor_dark']
+        },
+        yearTitle: {
+            class: ['text-textColor_light dark:text-textColor_dark']
+        },
+        nextbutton: {
+            class: ['text-textColor_light dark:text-textColor_dark']
+        },
+        weekday: {
+            class: ['text-textColor_light dark:text-textColor_dark']
+        },
+        daylabel: ({ context }) => ({
+            class: [
+                {
+                    'text-textColor_light dark:text-textColor_dark': !context.selected && !context.disabled && !context.date.today,
+                    'text-primary_light dark:text-primary_dark ': context.selected && !context.disabled
+                },
+                'focus:outline-none focus:outline-offset-0 focus:ring focus:ring-primary_light dark:focus:ring-primary_dark ring-opacity-80',
+
+            ]
+        }),
     },
 
 
@@ -632,7 +662,25 @@ const PrimevueDesignPreset = {
                 'text-textColor_light dark:text-textColor_dark',
             ]}),
     },
+    dropdown: {
+        root: ({ props, state }) => ({
+            class: [
+
+                // States
+                'hover:border-primary_light dark:hover:border-primary_dark',
+                { 'outline-none outline-offset-0 ring ring-primary_light dark:focus:ring-primary_dark ring-opacity-80': state.focused },
+
+            ]
+        }),
+        item: ({ context }) => ({
+            class: [
+                'bg-tile_color_light dark:bg-tile_color_dark',
+                'border-b_color_light dark:border-b_color_dark',
+                'text-textColor_light dark:text-textColor_dark',
 
+            ]
+        }),
+    }
 }
 
 export default PrimevueDesignPreset

frontend/components/statistics/BarChart.vue

@@ -16,6 +16,14 @@ const props = defineProps({
     type: Object,
     default: () => {},
   },
+  labels: {
+    type: Array,
+    required: true
+  },
+  xAxisText: {
+    type: String,
+    default: 'Magnitude Range'
+  },
   isLoading: {
     type: Boolean,
     default: false,
@@ -40,7 +48,7 @@ watch(
 const processData = () => {
   if (!props.data) return
 
-
+  /*
   const labelCounts = {
     '0-2': 0,
     '2-4': 0,
@@ -50,24 +58,29 @@ const processData = () => {
     'Unknown': 0
   };
 
+   */
+
+  const labelCounts: {[key: string]: number} = props.labels?.reduce((acc: any, label: string) => {
+    acc[label] = 0
+    return acc
+  }, {})
   Object.keys(labelCounts).forEach(key => {
     const data = props.data[key]
     if (data) {
       labelCounts[key] = data
     }
   })
 
-
   // Update the dataset with the counts
   chartData.value.datasets[0].data = Object.values(labelCounts);
 }
 
 const setChartData = () => {
   return {
-    labels: ['0-2', '2-4', '4-6', '6-8', '8-10', 'Unknown'],
+    labels: props.labels,
     datasets: [
       {
-        label: 'Mag Range Count',
+        label: props.xAxisText + ' Count',
         backgroundColor: '#009b91',
         data: []
       }
@@ -88,7 +101,7 @@ const setChartOptions = () => {
         position: 'bottom',
         title: {
           display: true,
-          text: 'Magnitude Range'
+          text: props.xAxisText
         }
       },
       y: {

frontend/components/statistics/EventDistribution.vue

@@ -1,17 +1,16 @@
 <template>
   <div>
-    <PrimeDataTable :value="data">
+    <PrimeDataTable :value="eventTypeData">
       <PrimeColumn field="type" header="Event Type" :sortable="true"></PrimeColumn>
       <PrimeColumn field="count" header="Occurrences" :sortable="true"></PrimeColumn>
       <PrimeColumn field="probability" header="Probability (%)" :sortable="true"></PrimeColumn>
     </PrimeDataTable>
-    <div style="height: 20px"></div>
-    <div>
+    <div class="mt-3">
       <PrimePanel header="Event Occurrence Probability (in a Row)">
 
       <div class="input-grid">
         <label for="typeDropdown" class="grid-label">Choose an event type:</label>
-        <PrimeDropdown v-model="selectedType" :options="data.map(item => item.type)" class="grid-input" />
+        <PrimeDropdown v-model="selectedType" :options="eventTypeData.map(item => item.type)" class="grid-input" />
 
         <label for="numberOfTimes" class="grid-label">Enter the number of occurrences:</label>
         <PrimeInputNumber v-model="numberOfTimes" :min="0" :max="1000" class="grid-input" />
@@ -26,7 +25,7 @@
       <div v-if="selectedType && numberOfTimes !== null && !isNaN(numberOfTimes)">
         <PrimePanel header="Result">
           <b>
-            The probability of {{ selectedType }} occurring exactly {{ numberOfTimes }} times in a row is: {{ calculateProbability(selectedType, numberOfTimes) }}%
+            The probability of event type <i>{{ selectedType }}</i> occurring exactly {{ numberOfTimes }} times in a row is: {{ calculateProbability(selectedType, numberOfTimes) }}%
           </b>
         </PrimePanel>
 
@@ -41,7 +40,15 @@ import { ref } from 'vue';
 
 
 
-const data = ref([
+const props = defineProps({
+    data : {
+      type: Object,
+      //default: () => ({type: "earthquake", count: 33083, probability: 97.3115})
+      required: true
+    }
+})
+
+const eventTypeData = ref([
   { type: "earthquake", count: 33083, probability: 97.3115 },
   { type: "ice quake", count: 355, probability: 1.0442 },
   { type: "quarry blast", count: 329, probability: 0.9677 },
@@ -50,6 +57,20 @@ const data = ref([
   { type: "other event", count: 3, probability: 0.0088 },
 ]);
 
+watch(() => props.data, () => {
+  eventTypeData.value = convertEventTypeObjectToArray(props.data)
+})
+
+onMounted(() => {
+  eventTypeData.value = convertEventTypeObjectToArray(props.data)
+})
+
+const convertEventTypeObjectToArray = (evenTypeObject: {[key: string]: { count: number, probability: number }}) => {
+  return Object.entries(evenTypeObject).map(([key, value]) => {
+    return { type: key, count: value.count, probability: value.probability * 100}
+  })
+}
+
 const columns = [
   { field: 'type', header: 'Type' },
   { field: 'count', header: 'Count' },
@@ -60,7 +81,7 @@ const selectedType = ref(null);
 const numberOfTimes = ref(5);
 
 const calculateProbability = (type, times) => {
-  const selectedRow = data.value.find(item => item.type === type);
+  const selectedRow = eventTypeData.value.find(item => item.type === type);
   if (selectedRow && times !== null && !isNaN(times)) {
     const n = times;
     const p = selectedRow.probability / 100;