Skip to content

Commit 40cc740

Browse files
joshsplunksjoshsplunks
committed
added prototype passive deployment, milvus and hashing encoder content
1 parent 5fc4910 commit 40cc740

22 files changed

+1001
-2
lines changed

‎beta_content/hashing_encoder_example/data/ui/views/hashing_encoder_dashboard.xml

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
<form version="1.1">
2+
<label>Hashing Encoder Dashboard</label>
3+
<search id="base_all">
4+
<query>source="whatismybrowser-user-agent-database.csv" host="wimb_sample" index="dsdl_testing_josh" sourcetype="wimb_csv"
5+
| table user_agent
6+
| fit MLTKContainer algo=hashing_encoder vector_length=$vector_length$ user_agent
7+
| rename predicted_* as encoded_*
8+
9+
</query>
10+
<earliest>0</earliest>
11+
<sampleRatio>1</sampleRatio>
12+
</search>
13+
<search id="umap_all">
14+
<query>source="whatismybrowser-user-agent-database.csv" host="wimb_sample" index="dsdl_testing_josh" sourcetype="wimb_csv"
15+
| table user_agent
16+
| fit MLTKContainer algo=hashing_encoder vector_length=$vector_length$ user_agent
17+
| rename predicted_* as encoded_* | fit MLTKContainer algo=umap encoded_*
18+
| rename predicted_* as embedded_*
19+
</query>
20+
</search>
21+
<fieldset submitButton="false">
22+
<input type="dropdown" token="vector_length">
23+
<label>Encoded Vector Length</label>
24+
<choice value="2">2</choice>
25+
<choice value="4">4</choice>
26+
<choice value="8">8</choice>
27+
<choice value="16">16</choice>
28+
<choice value="32">32</choice>
29+
<choice value="64">64</choice>
30+
<choice value="128">128</choice>
31+
<choice value="256">256</choice>
32+
<choice value="512">512</choice>
33+
<default>32</default>
34+
<initialValue>32</initialValue>
35+
</input>
36+
</fieldset>
37+
<row>
38+
<panel>
39+
<title>Encoding</title>
40+
<table>
41+
<search base="base_all">
42+
<query> | eval encoding=""
43+
| foreach encoded_* [eval encoding=encoding+tostring(&lt;&lt;FIELD&gt;&gt;)] | fields user_agent encoding</query>
44+
</search>
45+
<option name="count">15</option>
46+
<option name="dataOverlayMode">none</option>
47+
<option name="drilldown">cell</option>
48+
<option name="percentagesRow">false</option>
49+
<option name="refresh.display">progressbar</option>
50+
<option name="rowNumbers">false</option>
51+
<option name="totalsRow">false</option>
52+
<option name="wrap">true</option>
53+
<drilldown>
54+
<link target="_blank">/app/mltk-container/hashing_encoder_dashboard_reference_search?form.ref_ua=$row.user_agent$&amp;form.vector_length=$vector_length$</link>
55+
</drilldown>
56+
</table>
57+
</panel>
58+
<panel>
59+
<title>UMAP Of Encoded User Agents</title>
60+
<chart>
61+
<search base="umap_all">
62+
<query>| eval label="all", size=1
63+
| table label embedded_* size</query>
64+
</search>
65+
<option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
66+
<option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
67+
<option name="charting.axisTitleX.visibility">visible</option>
68+
<option name="charting.axisTitleY.visibility">visible</option>
69+
<option name="charting.axisTitleY2.visibility">visible</option>
70+
<option name="charting.axisX.abbreviation">none</option>
71+
<option name="charting.axisX.scale">linear</option>
72+
<option name="charting.axisY.abbreviation">none</option>
73+
<option name="charting.axisY.scale">linear</option>
74+
<option name="charting.axisY2.abbreviation">none</option>
75+
<option name="charting.axisY2.enabled">0</option>
76+
<option name="charting.axisY2.scale">inherit</option>
77+
<option name="charting.chart">bubble</option>
78+
<option name="charting.chart.bubbleMaximumSize">2</option>
79+
<option name="charting.chart.bubbleMinimumSize">1</option>
80+
<option name="charting.chart.bubbleSizeBy">area</option>
81+
<option name="charting.chart.markerSize">1</option>
82+
<option name="charting.chart.nullValueMode">gaps</option>
83+
<option name="charting.chart.showDataLabels">none</option>
84+
<option name="charting.chart.sliceCollapsingThreshold">0.01</option>
85+
<option name="charting.chart.stackMode">default</option>
86+
<option name="charting.chart.style">shiny</option>
87+
<option name="charting.drilldown">none</option>
88+
<option name="charting.layout.splitSeries">0</option>
89+
<option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
90+
<option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
91+
<option name="charting.legend.mode">standard</option>
92+
<option name="charting.legend.placement">none</option>
93+
<option name="charting.lineWidth">2</option>
94+
<option name="height">768</option>
95+
<option name="refresh.display">progressbar</option>
96+
<option name="trellis.enabled">0</option>
97+
<option name="trellis.scales.shared">1</option>
98+
<option name="trellis.size">medium</option>
99+
</chart>
100+
</panel>
101+
</row>
102+
</form>

‎beta_content/hashing_encoder_example/data/ui/views/hashing_encoder_dashboard_clustering.xml

+75
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
<form version="1.1">
2+
<label>Hashing Encoder Dashboard: Clustering</label>
3+
<fieldset submitButton="false">
4+
<input type="dropdown" token="vector_length">
5+
<label>Encoded Vector Length</label>
6+
<choice value="2">2</choice>
7+
<choice value="4">4</choice>
8+
<choice value="8">8</choice>
9+
<choice value="16">16</choice>
10+
<choice value="32">32</choice>
11+
<choice value="64">64</choice>
12+
<choice value="128">128</choice>
13+
<choice value="256">256</choice>
14+
<choice value="512">512</choice>
15+
<default>32</default>
16+
<initialValue>32</initialValue>
17+
</input>
18+
</fieldset>
19+
<row>
20+
<panel>
21+
<title>UMAP Of Encoded User Agents</title>
22+
<chart>
23+
<search>
24+
<query>source="whatismybrowser-user-agent-database.csv" host="wimb_sample" index="dsdl_testing_josh" sourcetype="wimb_csv"
25+
| head 10000
26+
| table user_agent
27+
| fit MLTKContainer algo=hashing_encoder vector_length=$vector_length$ user_agent
28+
| rename predicted_* as encoded_*
29+
| fit MLTKContainer algo=umap encoded_*
30+
| rename predicted_* as embedded_*
31+
| fields - encoded_*
32+
| fit DBSCAN eps=1 embedded_*
33+
| eval label=cluster, size=0.2
34+
| table label embedded_* size</query>
35+
<earliest>$earliest$</earliest>
36+
<latest>$latest$</latest>
37+
</search>
38+
<option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
39+
<option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
40+
<option name="charting.axisTitleX.visibility">visible</option>
41+
<option name="charting.axisTitleY.visibility">visible</option>
42+
<option name="charting.axisTitleY2.visibility">visible</option>
43+
<option name="charting.axisX.abbreviation">none</option>
44+
<option name="charting.axisX.scale">linear</option>
45+
<option name="charting.axisY.abbreviation">none</option>
46+
<option name="charting.axisY.scale">linear</option>
47+
<option name="charting.axisY2.abbreviation">none</option>
48+
<option name="charting.axisY2.enabled">0</option>
49+
<option name="charting.axisY2.scale">inherit</option>
50+
<option name="charting.chart">bubble</option>
51+
<option name="charting.chart.bubbleMaximumSize">2</option>
52+
<option name="charting.chart.bubbleMinimumSize">1</option>
53+
<option name="charting.chart.bubbleSizeBy">area</option>
54+
<option name="charting.chart.markerSize">1</option>
55+
<option name="charting.chart.nullValueMode">gaps</option>
56+
<option name="charting.chart.showDataLabels">none</option>
57+
<option name="charting.chart.sliceCollapsingThreshold">0.01</option>
58+
<option name="charting.chart.stackMode">default</option>
59+
<option name="charting.chart.style">shiny</option>
60+
<option name="charting.drilldown">none</option>
61+
<option name="charting.layout.splitSeries">0</option>
62+
<option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
63+
<option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
64+
<option name="charting.legend.mode">standard</option>
65+
<option name="charting.legend.placement">none</option>
66+
<option name="charting.lineWidth">2</option>
67+
<option name="height">768</option>
68+
<option name="refresh.display">progressbar</option>
69+
<option name="trellis.enabled">0</option>
70+
<option name="trellis.scales.shared">1</option>
71+
<option name="trellis.size">medium</option>
72+
</chart>
73+
</panel>
74+
</row>
75+
</form>

‎beta_content/hashing_encoder_example/data/ui/views/hashing_encoder_dashboard_reference_search.xml

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
<form version="1.1">
2+
<label>Hashing Encoder Dashboard: Reference Search</label>
3+
<fieldset submitButton="false">
4+
<input type="dropdown" token="vector_length">
5+
<label>Encoded Vector Length</label>
6+
<choice value="2">2</choice>
7+
<choice value="4">4</choice>
8+
<choice value="8">8</choice>
9+
<choice value="16">16</choice>
10+
<choice value="32">32</choice>
11+
<choice value="64">64</choice>
12+
<choice value="128">128</choice>
13+
<choice value="256">256</choice>
14+
<choice value="512">512</choice>
15+
<default>32</default>
16+
<initialValue>32</initialValue>
17+
</input>
18+
<input type="text" token="ref_ua" searchWhenChanged="true">
19+
<label>Reference User-agent</label>
20+
</input>
21+
</fieldset>
22+
<row>
23+
<panel>
24+
<title>Similar User Agents to Reference: Showing Differences</title>
25+
<table>
26+
<title>Reference UA: $ref_ua$</title>
27+
<search>
28+
<query>source="whatismybrowser-user-agent-database.csv" host="wimb_sample" index="dsdl_testing_josh" sourcetype="wimb_csv"
29+
| table user_agent
30+
| fit MLTKContainer algo=hashing_encoder vector_length=$vector_length$ user_agent
31+
| rename predicted_* as encoded_*
32+
| fit MLTKContainer algo=hashing_encoder_distance comparison_string="$ref_ua$" vector_length=$vector_length$ return_vector=0 user_agent
33+
| rename predicted_distance as distance
34+
| sort distance
35+
| head 100
36+
| fit MLTKContainer algo=hashing_encoder_difference comparison_string="$ref_ua$" user_agent
37+
| table user_agent distance predicted_*</query>
38+
</search>
39+
<option name="count">5</option>
40+
<option name="dataOverlayMode">none</option>
41+
<option name="drilldown">none</option>
42+
<option name="percentagesRow">false</option>
43+
<option name="refresh.display">progressbar</option>
44+
<option name="rowNumbers">false</option>
45+
<option name="totalsRow">false</option>
46+
<option name="wrap">true</option>
47+
</table>
48+
</panel>
49+
<panel>
50+
<chart>
51+
<search>
52+
<query>source="whatismybrowser-user-agent-database.csv" host="wimb_sample" index="dsdl_testing_josh" sourcetype="wimb_csv"
53+
| table user_agent
54+
| fit MLTKContainer algo=hashing_encoder vector_length=$vector_length$ user_agent
55+
| rename predicted_* as encoded_*
56+
| fit MLTKContainer algo=hashing_encoder_distance comparison_string="$ref_ua$" vector_length=$vector_length$ return_vector=0 user_agent
57+
| rename predicted_distance as distance
58+
| sort distance
59+
| streamstats count
60+
| eval label=if(count&lt;20,"near","far")
61+
| eval label=if(distance==0,"reference",label)
62+
| eval size=1
63+
| eval size=if(label=="reference",5,size)
64+
| eval size=if(label=="near",3,size)
65+
| fit MLTKContainer algo=umap encoded_*
66+
| rename predicted_* as embedded_*
67+
| table label embedded_* size
68+
| sort label</query>
69+
<earliest>$earliest$</earliest>
70+
<latest>$latest$</latest>
71+
</search>
72+
<option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
73+
<option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
74+
<option name="charting.axisTitleX.visibility">visible</option>
75+
<option name="charting.axisTitleY.visibility">visible</option>
76+
<option name="charting.axisTitleY2.visibility">visible</option>
77+
<option name="charting.axisX.abbreviation">none</option>
78+
<option name="charting.axisX.scale">linear</option>
79+
<option name="charting.axisY.abbreviation">none</option>
80+
<option name="charting.axisY.scale">linear</option>
81+
<option name="charting.axisY2.abbreviation">none</option>
82+
<option name="charting.axisY2.enabled">0</option>
83+
<option name="charting.axisY2.scale">inherit</option>
84+
<option name="charting.chart">bubble</option>
85+
<option name="charting.chart.bubbleMaximumSize">3</option>
86+
<option name="charting.chart.bubbleMinimumSize">2</option>
87+
<option name="charting.chart.bubbleSizeBy">area</option>
88+
<option name="charting.chart.markerSize">1</option>
89+
<option name="charting.chart.nullValueMode">gaps</option>
90+
<option name="charting.chart.showDataLabels">none</option>
91+
<option name="charting.chart.sliceCollapsingThreshold">0.01</option>
92+
<option name="charting.chart.stackMode">default</option>
93+
<option name="charting.chart.style">shiny</option>
94+
<option name="charting.drilldown">none</option>
95+
<option name="charting.layout.splitSeries">0</option>
96+
<option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
97+
<option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
98+
<option name="charting.legend.mode">standard</option>
99+
<option name="charting.legend.placement">right</option>
100+
<option name="charting.lineWidth">2</option>
101+
<option name="height">780</option>
102+
<option name="refresh.display">progressbar</option>
103+
<option name="trellis.enabled">0</option>
104+
<option name="trellis.scales.shared">1</option>
105+
<option name="trellis.size">medium</option>
106+
</chart>
107+
</panel>
108+
</row>
109+
</form>

‎beta_content/passive_deployment_prototypes/prototpye_milvus_example/README.md

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
Usage steps to deploy on a docker host:
2+
- Install docker https://docs.docker.com/engine/install/ (Docker desktop not required)
3+
- Install docker-compose https://docs.docker.com/compose/install/
4+
- Read dsdl_deploy_static_dev_container.sh to ensure you are happy with what we're configuring.
5+
- Run dsdl_deploy_static_dev_container.sh as root or with sufficient privs
6+
- Stop Splunk on the search head
7+
- Copy the contents of ./conf_files/ from the docker host into splunk/etc/apps/mltk-container/local/ on the serch head
8+
- Start Splunk on the search head
9+
- Check the dev container appears with details in the Containers page in the DSDL UI
10+
- Run a simple example to validate that fit/apply works
11+
12+
Note:
13+
- This template uses the default container configuration which uses an insecure certificate, this may or may not be fine for your test environment but should not be used in production.
14+
- This template only deploys a dev container, future guidance will detail how to configure dev and prod containers using dynamic templates and deploy them statically in docker and in K8s using HELM.
15+
- If you stop or restart the environment you will need to update the config on the splunk search head
16+
17+
Routing requirements:
18+
- Splunk search head must be able to access port 5000 or appropriately mapped ingress on the docker host for API communication
19+
- Users must be able to access appropriate ports or ingress on the docker host to access web applciations hosted in the container
20+
- Jupyter:8888 (Required)
21+
- Mlflow:6060 (Optional)
22+
- Spark:4040 (Optional)
23+
- Tensorboard:6006 (Optional)

‎beta_content/passive_deployment_prototypes/prototpye_milvus_example/compose_files/dsdl-base.yml

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
services:
2+
dsdl-base:
3+
container_name: dsdl-base
4+
image: phdrieger/mltk-container-golden-image-cpu:5.1.0
5+
volumes:
6+
- mltk-container-data:/srv:rw
7+
- mltk-container-app:/srv/backup/app:ro
8+
- mltk-container-notebooks:/srv/backup/notebooks:ro
9+
ports:
10+
- 5000:5000 # Model Interaction API
11+
environment:
12+
- olly_enabled=false
13+
- splunk_access_enabled=false
14+
- splunk_hec_enabled=false
15+
- ENABLE_HTTPS=true
16+
- JUPYTER_PASSWD=sha1:f7432152c71d:e8520c26b9d960e838d562768c1d24ef5b9b76c7
17+
- MODE_DEV_PROD=PROD

0 commit comments

Comments
 (0)