fix: updating ipv6-v4 regex in data models (#831)

Current regex for data models support only basic ipv6 regex.

Support for advanced representation of ipv6 is required, hence this pr
supports that.

ref: ipv6 represenation -
https://en.wikipedia.org/wiki/IPv6#Address_representation

Tested it with gcp addon which as combination of ipv4 and v6 samples in
host fields, also tested it on
splunk/splunk-add-on-for-okta-identity-cloud#261
where advanced ipv6 represenation is present

Added e2e test which tests extraction of some ipv6 formats using src_ip
and dest_ip fields in network_traffic data model

Tested regex with below samples:
Regex :
`(?:(?:::ffff:)|(?:[0-9a-fA-F]{1,4}:){6}ffff:)\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?>([a-f0-9]{1,4})(?>:(?1)){7}|(?!(?:.[a-f0-9](?>:|$)){8,})^((?1)(?>:(?1)){0,6})?::(?2)?(?!(?:.[a-f0-9](?>:|$))))|(?>(?>(?1)(?>:(?1)){5}:|(?!(?:.*[a-f0-9]:){6,})(?3)?::(?>((?1)(?>:(?1)){0,4}):)?)?(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?>\.(?4)){3})`

1234::
2001:db8::
::1334
::
::1325
2001:0db8::1:2:3456
2001::1:2:3
2001:db8::1:2
::ffff:192.168.1.1
::
::ffff:192.168.1.112
::1
2001:0db8::1:2:3
0000:0000:0000:0000:0000:0000:0000:0001
ff02:0000:0000:0000:0000:0000:0000:0001
fe80:0000:0000:0000:a299:9bff:fe18:50d1
2001:0db8:1111:000a:00b0:0000:9000:0200
2001:0db8:0000:0000:abcd:0000:0000:1234
2001:0db8:cafe:0001:0000:0000:0000:0100
2001:0db8:cafe:0001:0000:0000:0000:0200
2001:db80:1000:a000:0000:bc00:abcd:d0b0
1:2:3:4:5:6:7::
fe80::a299:9bff:fe18:50d1
::3212
::1212
2001::abcd::1234
2001:db80:1000:a000:0000:bc00:abcd:d0b0
2001::abcd
2001:0000:0000:0000:abcd:0000:0000:1234
2001:0000:0000:abcd:0000:0000:0000:1234
2001:0000:abcd:0000:0000:0000:0000:1234
2001:db8:1111:a:b0:0:9000:200
fe80:0:0:0:a299:9bff:fe18:50d1
0:0:0:0:0:0:0:1
0000:0000:0000:0000:0000:0000:0000:0001
0:0:0:0:0:0:ffff:192.168.10.10
2001:0000:0000:0000:0000:abcd:0000:1
::ffff:192.168.10.10
2001:0db8::1:2:3

Author: harshilgajera-crest

.github/workflows/build-test-release.yml

@@ -170,6 +170,7 @@ jobs:
           "splunk_setup_fixture",
           "splunk_app_req",
           "splunk_app_req_broken",
+          "splunk_cim_model_ipv6_regex",
         ]
     steps:
       - uses: actions/checkout@v4

Dockerfile.tests

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-FROM ubuntu:latest
+FROM ubuntu:22.04
 
 RUN mkdir -p /work/tests
 RUN mkdir -p /work/test-results/functional

pytest_splunk_addon/standard_lib/data_models/Intrusion_Detection.json

@@ -23,7 +23,7 @@
           "name": "dest",
           "type": "conditional",
           "condition": "ids_type=\"network\"",
-          "validity": "case(in(upper(transport), \"HOPOPT\", \"ICMP\", \"IGMP\", \"GGP\", \"IP-IN-IP\", \"ST\", \"TCP\", \"CBT\", \"EGP\", \"IGP\", \"BBN-RCC-MON\", \"NVP-II\", \"PUP\", \"ARGUS\", \"EMCON\", \"XNET\", \"CHAOS\", \"UDP\", \"MUX\", \"DCN-MEAS\", \"HMP\", \"PRM\", \"XNS-ID\", \"TRUNK-1\", \"TRUNK-2\", \"LEAF-1\", \"LEAF-2\", \"RDP\", \"IRTP\", \"ISO-TP4\", \"NETBLT\", \"MFE-NSP\", \"MERIT-INP\", \"DCCP\", \"3CP\", \"IDPR\", \"XTP\", \"DDP\", \"IDPR-CMTP\", \"TP++\", \"IL\", \"IPV6\", \"SDRP\", \"IPV6-ROUTE\", \"IPV6-FRAG\", \"IDRP\", \"RSVP\", \"GRES\", \"DSR\", \"BNA\", \"ESP\", \"AH\", \"I-NLSP\", \"SWIPE\", \"NARP\", \"MOBILE\", \"TLSP\", \"SKIP\", \"IPV6-ICMP\", \"IPC6-NONXT\", \"IPV6-OPTS\", \"CFTP\", \"SAT-EXPAK\", \"KRYPTOLAN\", \"RVD\", \"IPPC\", \"SAT-MON\", \"VISA\", \"IPCU\", \"CPNX\", \"CPHB\", \"WSN\", \"PVP\", \"BR-SAT-MON\", \"SUN-ND\", \"WB-MON\", \"WB-EXPAK\", \"ISO-IP\", \"VMTP\", \"SECURE-VMTP\", \"VINES\", \"TTP\", \"IPTM\", \"NSFNET-IGP\", \"DGP\", \"TCF\", \"EIGRP\", \"OSPF\", \"SPRITE-RPC\", \"LARP\", \"MTP\", \"AX.25\", \"OS\", \"MICP\", \"SCC-SP\", \"ETHERIP\", \"ENCAP\", \"GMTP\", \"IFMP\", \"PNNI\", \"PIM\", \"ARIS\", \"SCPS\", \"QNX\", \"A/N\", \"IPCOMP\", \"SNP\", \"COMPAQ-PEER\", \"IPX-IN-IP\", \"VRRP\", \"PGM\", \"L2TP\", \"DDX\", \"IATP\", \"STP\", \"SRP\", \"UTI\", \"SMP\", \"SM\", \"PTP\", \"IS-IS OVER IPV4\", \"FIRE\", \"CRTP\", \"CRUDP\", \"SSCOPMCE\", \"IPLT\", \"SPS\", \"PIPE\", \"SCTP\", \"FC\", \"RSVP-E2E-IGNORE\", \"MOBILITY HEADER\", \"UDPLITE\", \"MPLS-IN-IP\", \"MANET\", \"HIP\", \"SHIM6\", \"WESP\", \"ROHC\", \"ETHERNET\"), if(match(dest,\"(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}|(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\\.|$)){4}\"), dest, null()), match(dest,\"^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$\"), dest, true(), null())",
+          "validity": "case(in(upper(transport), \"HOPOPT\", \"ICMP\", \"IGMP\", \"GGP\", \"IP-IN-IP\", \"ST\", \"TCP\", \"CBT\", \"EGP\", \"IGP\", \"BBN-RCC-MON\", \"NVP-II\", \"PUP\", \"ARGUS\", \"EMCON\", \"XNET\", \"CHAOS\", \"UDP\", \"MUX\", \"DCN-MEAS\", \"HMP\", \"PRM\", \"XNS-ID\", \"TRUNK-1\", \"TRUNK-2\", \"LEAF-1\", \"LEAF-2\", \"RDP\", \"IRTP\", \"ISO-TP4\", \"NETBLT\", \"MFE-NSP\", \"MERIT-INP\", \"DCCP\", \"3CP\", \"IDPR\", \"XTP\", \"DDP\", \"IDPR-CMTP\", \"TP++\", \"IL\", \"IPV6\", \"SDRP\", \"IPV6-ROUTE\", \"IPV6-FRAG\", \"IDRP\", \"RSVP\", \"GRES\", \"DSR\", \"BNA\", \"ESP\", \"AH\", \"I-NLSP\", \"SWIPE\", \"NARP\", \"MOBILE\", \"TLSP\", \"SKIP\", \"IPV6-ICMP\", \"IPC6-NONXT\", \"IPV6-OPTS\", \"CFTP\", \"SAT-EXPAK\", \"KRYPTOLAN\", \"RVD\", \"IPPC\", \"SAT-MON\", \"VISA\", \"IPCU\", \"CPNX\", \"CPHB\", \"WSN\", \"PVP\", \"BR-SAT-MON\", \"SUN-ND\", \"WB-MON\", \"WB-EXPAK\", \"ISO-IP\", \"VMTP\", \"SECURE-VMTP\", \"VINES\", \"TTP\", \"IPTM\", \"NSFNET-IGP\", \"DGP\", \"TCF\", \"EIGRP\", \"OSPF\", \"SPRITE-RPC\", \"LARP\", \"MTP\", \"AX.25\", \"OS\", \"MICP\", \"SCC-SP\", \"ETHERIP\", \"ENCAP\", \"GMTP\", \"IFMP\", \"PNNI\", \"PIM\", \"ARIS\", \"SCPS\", \"QNX\", \"A/N\", \"IPCOMP\", \"SNP\", \"COMPAQ-PEER\", \"IPX-IN-IP\", \"VRRP\", \"PGM\", \"L2TP\", \"DDX\", \"IATP\", \"STP\", \"SRP\", \"UTI\", \"SMP\", \"SM\", \"PTP\", \"IS-IS OVER IPV4\", \"FIRE\", \"CRTP\", \"CRUDP\", \"SSCOPMCE\", \"IPLT\", \"SPS\", \"PIPE\", \"SCTP\", \"FC\", \"RSVP-E2E-IGNORE\", \"MOBILITY HEADER\", \"UDPLITE\", \"MPLS-IN-IP\", \"MANET\", \"HIP\", \"SHIM6\", \"WESP\", \"ROHC\", \"ETHERNET\"), if(match(dest,\"(?:(?:::ffff:)|(?:[0-9a-fA-F]{1,4}:){6}ffff:)\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}|(?>([a-f0-9]{1,4})(?>:(?1)){7}|(?!(?:.[a-f0-9](?>:|$)){8,})^((?1)(?>:(?1)){0,6})?::(?2)?(?!(?:.*[a-f0-9](?>:|$))))|(?>(?>(?1)(?>:(?1)){5}:|(?!(?:.*[a-f0-9]:){6,})(?3)?::(?>((?1)(?>:(?1)){0,4}):)?)?(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?>\\.(?4)){3})\"), dest, null()), match(dest,\"^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$\"), dest, true(), null())",
           "comment": "The destination of the attack detected by the intrusion detection system (IDS). You can alias this from more specific fields not included in this data model, such as dest_host, dest_ip, or dest_name."
         },
         {
@@ -76,7 +76,7 @@
           "name": "src",
           "type": "conditional",
           "condition": "ids_type=\"network\"",
-          "validity": "case(in(upper(transport), \"HOPOPT\", \"ICMP\", \"IGMP\", \"GGP\", \"IP-IN-IP\", \"ST\", \"TCP\", \"CBT\", \"EGP\", \"IGP\", \"BBN-RCC-MON\", \"NVP-II\", \"PUP\", \"ARGUS\", \"EMCON\", \"XNET\", \"CHAOS\", \"UDP\", \"MUX\", \"DCN-MEAS\", \"HMP\", \"PRM\", \"XNS-ID\", \"TRUNK-1\", \"TRUNK-2\", \"LEAF-1\", \"LEAF-2\", \"RDP\", \"IRTP\", \"ISO-TP4\", \"NETBLT\", \"MFE-NSP\", \"MERIT-INP\", \"DCCP\", \"3CP\", \"IDPR\", \"XTP\", \"DDP\", \"IDPR-CMTP\", \"TP++\", \"IL\", \"IPV6\", \"SDRP\", \"IPV6-ROUTE\", \"IPV6-FRAG\", \"IDRP\", \"RSVP\", \"GRES\", \"DSR\", \"BNA\", \"ESP\", \"AH\", \"I-NLSP\", \"SWIPE\", \"NARP\", \"MOBILE\", \"TLSP\", \"SKIP\", \"IPV6-ICMP\", \"IPC6-NONXT\", \"IPV6-OPTS\", \"CFTP\", \"SAT-EXPAK\", \"KRYPTOLAN\", \"RVD\", \"IPPC\", \"SAT-MON\", \"VISA\", \"IPCU\", \"CPNX\", \"CPHB\", \"WSN\", \"PVP\", \"BR-SAT-MON\", \"SUN-ND\", \"WB-MON\", \"WB-EXPAK\", \"ISO-IP\", \"VMTP\", \"SECURE-VMTP\", \"VINES\", \"TTP\", \"IPTM\", \"NSFNET-IGP\", \"DGP\", \"TCF\", \"EIGRP\", \"OSPF\", \"SPRITE-RPC\", \"LARP\", \"MTP\", \"AX.25\", \"OS\", \"MICP\", \"SCC-SP\", \"ETHERIP\", \"ENCAP\", \"GMTP\", \"IFMP\", \"PNNI\", \"PIM\", \"ARIS\", \"SCPS\", \"QNX\", \"A/N\", \"IPCOMP\", \"SNP\", \"COMPAQ-PEER\", \"IPX-IN-IP\", \"VRRP\", \"PGM\", \"L2TP\", \"DDX\", \"IATP\", \"STP\", \"SRP\", \"UTI\", \"SMP\", \"SM\", \"PTP\", \"IS-IS OVER IPV4\", \"FIRE\", \"CRTP\", \"CRUDP\", \"SSCOPMCE\", \"IPLT\", \"SPS\", \"PIPE\", \"SCTP\", \"FC\", \"RSVP-E2E-IGNORE\", \"MOBILITY HEADER\", \"UDPLITE\", \"MPLS-IN-IP\", \"MANET\", \"HIP\", \"SHIM6\", \"WESP\", \"ROHC\", \"ETHERNET\"), if(match(src,\"(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}|(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\\.|$)){4}\"), src, null()), match(src,\"^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$\"), src, true(), null())",
+          "validity": "case(in(upper(transport), \"HOPOPT\", \"ICMP\", \"IGMP\", \"GGP\", \"IP-IN-IP\", \"ST\", \"TCP\", \"CBT\", \"EGP\", \"IGP\", \"BBN-RCC-MON\", \"NVP-II\", \"PUP\", \"ARGUS\", \"EMCON\", \"XNET\", \"CHAOS\", \"UDP\", \"MUX\", \"DCN-MEAS\", \"HMP\", \"PRM\", \"XNS-ID\", \"TRUNK-1\", \"TRUNK-2\", \"LEAF-1\", \"LEAF-2\", \"RDP\", \"IRTP\", \"ISO-TP4\", \"NETBLT\", \"MFE-NSP\", \"MERIT-INP\", \"DCCP\", \"3CP\", \"IDPR\", \"XTP\", \"DDP\", \"IDPR-CMTP\", \"TP++\", \"IL\", \"IPV6\", \"SDRP\", \"IPV6-ROUTE\", \"IPV6-FRAG\", \"IDRP\", \"RSVP\", \"GRES\", \"DSR\", \"BNA\", \"ESP\", \"AH\", \"I-NLSP\", \"SWIPE\", \"NARP\", \"MOBILE\", \"TLSP\", \"SKIP\", \"IPV6-ICMP\", \"IPC6-NONXT\", \"IPV6-OPTS\", \"CFTP\", \"SAT-EXPAK\", \"KRYPTOLAN\", \"RVD\", \"IPPC\", \"SAT-MON\", \"VISA\", \"IPCU\", \"CPNX\", \"CPHB\", \"WSN\", \"PVP\", \"BR-SAT-MON\", \"SUN-ND\", \"WB-MON\", \"WB-EXPAK\", \"ISO-IP\", \"VMTP\", \"SECURE-VMTP\", \"VINES\", \"TTP\", \"IPTM\", \"NSFNET-IGP\", \"DGP\", \"TCF\", \"EIGRP\", \"OSPF\", \"SPRITE-RPC\", \"LARP\", \"MTP\", \"AX.25\", \"OS\", \"MICP\", \"SCC-SP\", \"ETHERIP\", \"ENCAP\", \"GMTP\", \"IFMP\", \"PNNI\", \"PIM\", \"ARIS\", \"SCPS\", \"QNX\", \"A/N\", \"IPCOMP\", \"SNP\", \"COMPAQ-PEER\", \"IPX-IN-IP\", \"VRRP\", \"PGM\", \"L2TP\", \"DDX\", \"IATP\", \"STP\", \"SRP\", \"UTI\", \"SMP\", \"SM\", \"PTP\", \"IS-IS OVER IPV4\", \"FIRE\", \"CRTP\", \"CRUDP\", \"SSCOPMCE\", \"IPLT\", \"SPS\", \"PIPE\", \"SCTP\", \"FC\", \"RSVP-E2E-IGNORE\", \"MOBILITY HEADER\", \"UDPLITE\", \"MPLS-IN-IP\", \"MANET\", \"HIP\", \"SHIM6\", \"WESP\", \"ROHC\", \"ETHERNET\"), if(match(src,\"(?:(?:::ffff:)|(?:[0-9a-fA-F]{1,4}:){6}ffff:)\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}|(?>([a-f0-9]{1,4})(?>:(?1)){7}|(?!(?:.[a-f0-9](?>:|$)){8,})^((?1)(?>:(?1)){0,6})?::(?2)?(?!(?:.*[a-f0-9](?>:|$))))|(?>(?>(?1)(?>:(?1)){5}:|(?!(?:.*[a-f0-9]:){6,})(?3)?::(?>((?1)(?>:(?1)){0,4}):)?)?(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?>\\.(?4)){3})\"), src, null()), match(src,\"^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$\"), src, true(), null())",
           "comment": "The source involved in the attack detected by the IDS. You can alias this from more specific fields not included in this data model, such as src_host, src_ip, or src_name."
         },
         {

pytest_splunk_addon/standard_lib/data_models/Network_Resolution.json

@@ -38,7 +38,7 @@
       {
         "name": "dest",
         "type": "required",
-        "validity": "case(in(upper(transport), \"TCP\", \"UDP\"), if(match(dest,\"(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}|(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\\.|$)){4}\"), dest, null()), match(dest,\"^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$\"), dest, true(), null())",
+        "validity": "case(in(upper(transport), \"TCP\", \"UDP\"), if(match(dest,\"(?:(?:::ffff:)|(?:[0-9a-fA-F]{1,4}:){6}ffff:)\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}|(?>([a-f0-9]{1,4})(?>:(?1)){7}|(?!(?:.[a-f0-9](?>:|$)){8,})^((?1)(?>:(?1)){0,6})?::(?2)?(?!(?:.*[a-f0-9](?>:|$))))|(?>(?>(?1)(?>:(?1)){5}:|(?!(?:.*[a-f0-9]:){6,})(?3)?::(?>((?1)(?>:(?1)){0,4}):)?)?(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?>\\.(?4)){3})\"), dest, null()), match(dest,\"^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$\"), dest, true(), null())",
         "comment": "The destination of the network resolution event. You can alias this from more specific fields, such as dest_host, dest_ip, or dest_name."
       },
       {
@@ -151,7 +151,7 @@
       {
         "name": "src",
         "type": "required",
-        "validity": "case(in(upper(transport), \"TCP\", \"UDP\"), if(match(src,\"(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}|(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\\.|$)){4}\"), src, null()), match(src,\"^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$\"), src, true(), null())",
+        "validity": "case(in(upper(transport), \"TCP\", \"UDP\"), if(match(src,\"(?:(?:::ffff:)|(?:[0-9a-fA-F]{1,4}:){6}ffff:)\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}|(?>([a-f0-9]{1,4})(?>:(?1)){7}|(?!(?:.[a-f0-9](?>:|$)){8,})^((?1)(?>:(?1)){0,6})?::(?2)?(?!(?:.*[a-f0-9](?>:|$))))|(?>(?>(?1)(?>:(?1)){5}:|(?!(?:.*[a-f0-9]:){6,})(?3)?::(?>((?1)(?>:(?1)){0,4}):)?)?(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?>\\.(?4)){3})\"), src, null()), match(src,\"^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$\"), src, true(), null())",
         "comment": "The source of the network resolution event. You can alias this from more specific fields, such as src_host, src_ip, or src_name."
       },
       {