Skip to content

Commit

Permalink
[exporter/splunkhec] Increase the performance of JSON marshaling (ope…
Browse files Browse the repository at this point in the history
…n-telemetry#34011)

**Description:**
Switch JSON libs to get more performance out of the JSON marshaler used
to marshal events.

Before:
```
goarch: arm64
pkg: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/splunkhecexporter
Benchmark_pushLogData_10_10_1024-10                                        11065            106969 ns/op           85002 B/op       1285 allocs/op
Benchmark_pushLogData_10_10_8K-10                                          12757             94831 ns/op           74142 B/op       1120 allocs/op
Benchmark_pushLogData_10_10_2M-10                                          13096             92878 ns/op           73534 B/op       1109 allocs/op
Benchmark_pushLogData_10_200_2M-10                                           650           1797755 ns/op         1460043 B/op      22010 allocs/op
Benchmark_pushLogData_100_200_2M-10                                           63          16883869 ns/op        14664365 B/op     220024 allocs/op
Benchmark_pushLogData_100_200_5M-10                                           61          16599464 ns/op        14769204 B/op     220013 allocs/op
Benchmark_pushLogData_compressed_10_10_1024-10                              1755            710944 ns/op         3102954 B/op       1193 allocs/op
Benchmark_pushLogData_compressed_10_10_8K-10                                6984            167371 ns/op           75065 B/op       1109 allocs/op
Benchmark_pushLogData_compressed_10_10_2M-10                                7285            165637 ns/op           74095 B/op       1109 allocs/op
Benchmark_pushLogData_compressed_10_200_2M-10                                334           3591145 ns/op         1465106 B/op      22010 allocs/op
Benchmark_pushLogData_compressed_100_200_2M-10                                31          36341606 ns/op        14599767 B/op     220015 allocs/op
Benchmark_pushLogData_compressed_100_200_5M-10                                31          36417519 ns/op        14598477 B/op     220014 allocs/op
Benchmark_pushMetricData_10_10_1024-10                                      8422            142076 ns/op           87314 B/op       1810 allocs/op
Benchmark_pushMetricData_10_10_8K-10                                        8280            143053 ns/op           94532 B/op       1810 allocs/op
Benchmark_pushMetricData_10_10_2M-10                                        4083            300601 ns/op         2185220 B/op       1812 allocs/op
Benchmark_pushMetricData_10_200_2M-10                                        397           3053150 ns/op         3818987 B/op      36012 allocs/op
Benchmark_pushMetricData_100_200_2M-10                                        37          33856887 ns/op        21432556 B/op     360034 allocs/op
Benchmark_pushMetricData_100_200_5M-10                                        30          41485322 ns/op        27750779 B/op     360034 allocs/op
Benchmark_pushMetricData_compressed_10_10_1024-10                           3810            309415 ns/op           89075 B/op       1810 allocs/op
Benchmark_pushMetricData_compressed_10_10_8K-10                             3828            307569 ns/op           95172 B/op       1810 allocs/op
Benchmark_pushMetricData_compressed_10_10_2M-10                             2688            453805 ns/op         2184373 B/op       1811 allocs/op
Benchmark_pushMetricData_compressed_10_200_2M-10                             177           6467414 ns/op         3815937 B/op      36012 allocs/op
Benchmark_pushMetricData_compressed_100_200_2M-10                             16          66096492 ns/op        19305614 B/op     360017 allocs/op
Benchmark_pushMetricData_compressed_100_200_5M-10                             16          65041961 ns/op        22450356 B/op     360017 allocs/op
Benchmark_pushMetricData_10_10_1024_MultiMetric-10                          3320            366943 ns/op          249636 B/op       3934 allocs/op
Benchmark_pushMetricData_10_10_8K_MultiMetric-10                            3319            361330 ns/op          249566 B/op       3934 allocs/op
Benchmark_pushMetricData_10_10_2M_MultiMetric-10                            3318            358159 ns/op          249583 B/op       3934 allocs/op
Benchmark_pushMetricData_10_200_2M_MultiMetric-10                            165           7502432 ns/op         4989958 B/op      78073 allocs/op
Benchmark_pushMetricData_100_200_2M_MultiMetric-10                            16          69800424 ns/op        49863629 B/op     780369 allocs/op
Benchmark_pushMetricData_100_200_5M_MultiMetric-10                            15          79718683 ns/op        49880688 B/op     780368 allocs/op
Benchmark_pushMetricData_compressed_10_10_1024_MultiMetric-10               2259            539714 ns/op          250326 B/op       3934 allocs/op
Benchmark_pushMetricData_compressed_10_10_8K_MultiMetric-10                 2360            540889 ns/op          250269 B/op       3934 allocs/op
Benchmark_pushMetricData_compressed_10_10_2M_MultiMetric-10                 2355            505609 ns/op          249576 B/op       3934 allocs/op
Benchmark_pushMetricData_compressed_10_200_2M_MultiMetric-10                 100          10467474 ns/op         5000816 B/op      78072 allocs/op
Benchmark_pushMetricData_compressed_100_200_2M_MultiMetric-10                 10         103846029 ns/op        49736248 B/op     780363 allocs/op
Benchmark_pushMetricData_compressed_100_200_5M_MultiMetric-10                 10         102504771 ns/op        49734080 B/op     780358 allocs/op
BenchmarkConsumeLogsRejected-10                                             1354            903909 ns/op          731212 B/op      11012 allocs/op
```

After:
```
goarch: arm64
pkg: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/splunkhecexporter
Benchmark_pushLogData_10_10_1024-10                                        14031             84592 ns/op           74842 B/op        821 allocs/op
Benchmark_pushLogData_10_10_8K-10                                          16302             72281 ns/op           65259 B/op        716 allocs/op
Benchmark_pushLogData_10_10_2M-10                                          16766             71721 ns/op           64664 B/op        709 allocs/op
Benchmark_pushLogData_10_200_2M-10                                           824           1448491 ns/op         1314978 B/op      14014 allocs/op
Benchmark_pushLogData_100_200_2M-10                                           73          13899896 ns/op        13208607 B/op     140034 allocs/op
Benchmark_pushLogData_100_200_5M-10                                           78          13690774 ns/op        13282520 B/op     140024 allocs/op
Benchmark_pushLogData_compressed_10_10_1024-10                              1940            570420 ns/op         2538497 B/op        784 allocs/op
Benchmark_pushLogData_compressed_10_10_8K-10                                8893            133297 ns/op           66024 B/op        709 allocs/op
Benchmark_pushLogData_compressed_10_10_2M-10                                8931            134021 ns/op           65465 B/op        709 allocs/op
Benchmark_pushLogData_compressed_10_200_2M-10                                385           3116794 ns/op         1315532 B/op      14015 allocs/op
Benchmark_pushLogData_compressed_100_200_2M-10                                37          30985287 ns/op        13149922 B/op     140035 allocs/op
Benchmark_pushLogData_compressed_100_200_5M-10                                38          31037704 ns/op        13149626 B/op     140033 allocs/op
Benchmark_pushMetricData_10_10_1024-10                                      7464            152713 ns/op           83334 B/op       1410 allocs/op
Benchmark_pushMetricData_10_10_8K-10                                        8011            152011 ns/op           90480 B/op       1410 allocs/op
Benchmark_pushMetricData_10_10_2M-10                                        3855            327686 ns/op         2181771 B/op       1417 allocs/op
Benchmark_pushMetricData_10_200_2M-10                                        367           3238910 ns/op         3735843 B/op      28020 allocs/op
Benchmark_pushMetricData_100_200_2M-10                                        37          30327247 ns/op        20711030 B/op     280055 allocs/op
Benchmark_pushMetricData_100_200_5M-10                                        37          30944880 ns/op        27001581 B/op     280051 allocs/op
Benchmark_pushMetricData_compressed_10_10_1024-10                           4766            250411 ns/op           84060 B/op       1410 allocs/op
Benchmark_pushMetricData_compressed_10_10_8K-10                             4627            245920 ns/op           91252 B/op       1410 allocs/op
Benchmark_pushMetricData_compressed_10_10_2M-10                             2776            411300 ns/op         2185072 B/op       1418 allocs/op
Benchmark_pushMetricData_compressed_10_200_2M-10                             246           4832722 ns/op         3737099 B/op      28022 allocs/op
Benchmark_pushMetricData_compressed_100_200_2M-10                             25          46085088 ns/op        18552164 B/op     280037 allocs/op
Benchmark_pushMetricData_compressed_100_200_5M-10                             25          45760102 ns/op        21754765 B/op     280037 allocs/op
Benchmark_pushMetricData_10_10_1024_MultiMetric-10                          4110            284592 ns/op          153998 B/op       1835 allocs/op
Benchmark_pushMetricData_10_10_8K_MultiMetric-10                            4045            284535 ns/op          154088 B/op       1835 allocs/op
Benchmark_pushMetricData_10_10_2M_MultiMetric-10                            4251            288325 ns/op          154016 B/op       1835 allocs/op
Benchmark_pushMetricData_10_200_2M_MultiMetric-10                            206           5787823 ns/op         3077042 B/op      36068 allocs/op
Benchmark_pushMetricData_100_200_2M_MultiMetric-10                            19          57907099 ns/op        30804149 B/op     360348 allocs/op
Benchmark_pushMetricData_100_200_5M_MultiMetric-10                            19          57568178 ns/op        30805770 B/op     360359 allocs/op
Benchmark_pushMetricData_compressed_10_10_1024_MultiMetric-10               3140            370639 ns/op          154425 B/op       1835 allocs/op
Benchmark_pushMetricData_compressed_10_10_8K_MultiMetric-10                 3278            368228 ns/op          154187 B/op       1835 allocs/op
Benchmark_pushMetricData_compressed_10_10_2M_MultiMetric-10                 3106            375295 ns/op          154715 B/op       1835 allocs/op
Benchmark_pushMetricData_compressed_10_200_2M_MultiMetric-10                 168           7128948 ns/op         3077226 B/op      36067 allocs/op
Benchmark_pushMetricData_compressed_100_200_2M_MultiMetric-10                 15          70611422 ns/op        30676065 B/op     360371 allocs/op
Benchmark_pushMetricData_compressed_100_200_5M_MultiMetric-10                 15          70289731 ns/op        30673553 B/op     360355 allocs/op
BenchmarkConsumeLogsRejected-10                                             1681            716325 ns/op          655652 B/op       7013 allocs/op
PASS
ok      github.com/open-telemetry/opentelemetry-collector-contrib/exporter/splunkhecexporter    49.913s
```

Benchstat
```
goarch: arm64
pkg: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/splunkhecexporter
                                                     │  before.txt   │               after.txt               │
                                                     │    sec/op     │    sec/op     vs base                 │
_pushLogData_10_10_1024-10                             106.97µ ± ∞ ¹   84.59µ ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_10_10_8K-10                                94.83µ ± ∞ ¹   72.28µ ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_10_10_2M-10                                92.88µ ± ∞ ¹   71.72µ ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_10_200_2M-10                               1.798m ± ∞ ¹   1.448m ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_100_200_2M-10                              16.88m ± ∞ ¹   13.90m ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_100_200_5M-10                              16.60m ± ∞ ¹   13.69m ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_compressed_10_10_1024-10                   710.9µ ± ∞ ¹   570.4µ ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_compressed_10_10_8K-10                     167.4µ ± ∞ ¹   133.3µ ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_compressed_10_10_2M-10                     165.6µ ± ∞ ¹   134.0µ ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_compressed_10_200_2M-10                    3.591m ± ∞ ¹   3.117m ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_compressed_100_200_2M-10                   36.34m ± ∞ ¹   30.99m ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_compressed_100_200_5M-10                   36.42m ± ∞ ¹   31.04m ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_10_1024-10                           142.1µ ± ∞ ¹   152.7µ ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_10_8K-10                             143.1µ ± ∞ ¹   152.0µ ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_10_2M-10                             300.6µ ± ∞ ¹   327.7µ ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_200_2M-10                            3.053m ± ∞ ¹   3.239m ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_100_200_2M-10                           33.86m ± ∞ ¹   30.33m ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_100_200_5M-10                           41.49m ± ∞ ¹   30.94m ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_10_1024-10                309.4µ ± ∞ ¹   250.4µ ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_10_8K-10                  307.6µ ± ∞ ¹   245.9µ ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_10_2M-10                  453.8µ ± ∞ ¹   411.3µ ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_200_2M-10                 6.467m ± ∞ ¹   4.833m ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_100_200_2M-10                66.10m ± ∞ ¹   46.09m ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_100_200_5M-10                65.04m ± ∞ ¹   45.76m ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_10_1024_MultiMetric-10               366.9µ ± ∞ ¹   284.6µ ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_10_8K_MultiMetric-10                 361.3µ ± ∞ ¹   284.5µ ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_10_2M_MultiMetric-10                 358.2µ ± ∞ ¹   288.3µ ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_200_2M_MultiMetric-10                7.502m ± ∞ ¹   5.788m ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_100_200_2M_MultiMetric-10               69.80m ± ∞ ¹   57.91m ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_100_200_5M_MultiMetric-10               79.72m ± ∞ ¹   57.57m ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_10_1024_MultiMetric-10    539.7µ ± ∞ ¹   370.6µ ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_10_8K_MultiMetric-10      540.9µ ± ∞ ¹   368.2µ ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_10_2M_MultiMetric-10      505.6µ ± ∞ ¹   375.3µ ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_200_2M_MultiMetric-10    10.467m ± ∞ ¹   7.129m ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_100_200_2M_MultiMetric-10   103.85m ± ∞ ¹   70.61m ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_100_200_5M_MultiMetric-10   102.50m ± ∞ ¹   70.29m ± ∞ ¹        ~ (p=1.000 n=1) ²
ConsumeLogsRejected-10                                  903.9µ ± ∞ ¹   716.3µ ± ∞ ¹        ~ (p=1.000 n=1) ²
geomean                                                 2.333m         1.876m        -19.60%
¹ need >= 6 samples for confidence interval at level 0.95
² need >= 4 samples to detect a difference at alpha level 0.05

                                                     │  before.txt   │               after.txt                │
                                                     │     B/op      │     B/op       vs base                 │
_pushLogData_10_10_1024-10                             83.01Ki ± ∞ ¹   73.09Ki ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_10_10_8K-10                               72.40Ki ± ∞ ¹   63.73Ki ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_10_10_2M-10                               71.81Ki ± ∞ ¹   63.15Ki ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_10_200_2M-10                              1.392Mi ± ∞ ¹   1.254Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_100_200_2M-10                             13.99Mi ± ∞ ¹   12.60Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_100_200_5M-10                             14.09Mi ± ∞ ¹   12.67Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_compressed_10_10_1024-10                  2.959Mi ± ∞ ¹   2.421Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_compressed_10_10_8K-10                    73.31Ki ± ∞ ¹   64.48Ki ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_compressed_10_10_2M-10                    72.36Ki ± ∞ ¹   63.93Ki ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_compressed_10_200_2M-10                   1.397Mi ± ∞ ¹   1.255Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_compressed_100_200_2M-10                  13.92Mi ± ∞ ¹   12.54Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_compressed_100_200_5M-10                  13.92Mi ± ∞ ¹   12.54Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_10_1024-10                          85.27Ki ± ∞ ¹   81.38Ki ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_10_8K-10                            92.32Ki ± ∞ ¹   88.36Ki ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_10_2M-10                            2.084Mi ± ∞ ¹   2.081Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_200_2M-10                           3.642Mi ± ∞ ¹   3.563Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_100_200_2M-10                          20.44Mi ± ∞ ¹   19.75Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_100_200_5M-10                          26.47Mi ± ∞ ¹   25.75Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_10_1024-10               86.99Ki ± ∞ ¹   82.09Ki ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_10_8K-10                 92.94Ki ± ∞ ¹   89.11Ki ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_10_2M-10                 2.083Mi ± ∞ ¹   2.084Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_200_2M-10                3.639Mi ± ∞ ¹   3.564Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_100_200_2M-10               18.41Mi ± ∞ ¹   17.69Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_100_200_5M-10               21.41Mi ± ∞ ¹   20.75Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_10_1024_MultiMetric-10              243.8Ki ± ∞ ¹   150.4Ki ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_10_8K_MultiMetric-10                243.7Ki ± ∞ ¹   150.5Ki ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_10_2M_MultiMetric-10                243.7Ki ± ∞ ¹   150.4Ki ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_200_2M_MultiMetric-10               4.759Mi ± ∞ ¹   2.934Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_100_200_2M_MultiMetric-10              47.55Mi ± ∞ ¹   29.38Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_100_200_5M_MultiMetric-10              47.57Mi ± ∞ ¹   29.38Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_10_1024_MultiMetric-10   244.5Ki ± ∞ ¹   150.8Ki ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_10_8K_MultiMetric-10     244.4Ki ± ∞ ¹   150.6Ki ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_10_2M_MultiMetric-10     243.7Ki ± ∞ ¹   151.1Ki ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_200_2M_MultiMetric-10    4.769Mi ± ∞ ¹   2.935Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_100_200_2M_MultiMetric-10   47.43Mi ± ∞ ¹   29.25Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_100_200_5M_MultiMetric-10   47.43Mi ± ∞ ¹   29.25Mi ± ∞ ¹        ~ (p=1.000 n=1) ²
ConsumeLogsRejected-10                                 714.1Ki ± ∞ ¹   640.3Ki ± ∞ ¹        ~ (p=1.000 n=1) ²
geomean                                                1.514Mi         1.229Mi        -18.87%
¹ need >= 6 samples for confidence interval at level 0.95
² need >= 4 samples to detect a difference at alpha level 0.05

                                                     │  before.txt   │               after.txt               │
                                                     │   allocs/op   │  allocs/op    vs base                 │
_pushLogData_10_10_1024-10                              1285.0 ± ∞ ¹    821.0 ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_10_10_8K-10                                1120.0 ± ∞ ¹    716.0 ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_10_10_2M-10                                1109.0 ± ∞ ¹    709.0 ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_10_200_2M-10                               22.01k ± ∞ ¹   14.01k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_100_200_2M-10                              220.0k ± ∞ ¹   140.0k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_100_200_5M-10                              220.0k ± ∞ ¹   140.0k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_compressed_10_10_1024-10                   1193.0 ± ∞ ¹    784.0 ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_compressed_10_10_8K-10                     1109.0 ± ∞ ¹    709.0 ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_compressed_10_10_2M-10                     1109.0 ± ∞ ¹    709.0 ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_compressed_10_200_2M-10                    22.01k ± ∞ ¹   14.02k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_compressed_100_200_2M-10                   220.0k ± ∞ ¹   140.0k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushLogData_compressed_100_200_5M-10                   220.0k ± ∞ ¹   140.0k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_10_1024-10                           1.810k ± ∞ ¹   1.410k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_10_8K-10                             1.810k ± ∞ ¹   1.410k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_10_2M-10                             1.812k ± ∞ ¹   1.417k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_200_2M-10                            36.01k ± ∞ ¹   28.02k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_100_200_2M-10                           360.0k ± ∞ ¹   280.1k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_100_200_5M-10                           360.0k ± ∞ ¹   280.1k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_10_1024-10                1.810k ± ∞ ¹   1.410k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_10_8K-10                  1.810k ± ∞ ¹   1.410k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_10_2M-10                  1.811k ± ∞ ¹   1.418k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_200_2M-10                 36.01k ± ∞ ¹   28.02k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_100_200_2M-10                360.0k ± ∞ ¹   280.0k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_100_200_5M-10                360.0k ± ∞ ¹   280.0k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_10_1024_MultiMetric-10               3.934k ± ∞ ¹   1.835k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_10_8K_MultiMetric-10                 3.934k ± ∞ ¹   1.835k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_10_2M_MultiMetric-10                 3.934k ± ∞ ¹   1.835k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_10_200_2M_MultiMetric-10                78.07k ± ∞ ¹   36.07k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_100_200_2M_MultiMetric-10               780.4k ± ∞ ¹   360.3k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_100_200_5M_MultiMetric-10               780.4k ± ∞ ¹   360.4k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_10_1024_MultiMetric-10    3.934k ± ∞ ¹   1.835k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_10_8K_MultiMetric-10      3.934k ± ∞ ¹   1.835k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_10_2M_MultiMetric-10      3.934k ± ∞ ¹   1.835k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_10_200_2M_MultiMetric-10     78.07k ± ∞ ¹   36.07k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_100_200_2M_MultiMetric-10    780.4k ± ∞ ¹   360.4k ± ∞ ¹        ~ (p=1.000 n=1) ²
_pushMetricData_compressed_100_200_5M_MultiMetric-10    780.4k ± ∞ ¹   360.4k ± ∞ ¹        ~ (p=1.000 n=1) ²
ConsumeLogsRejected-10                                 11.012k ± ∞ ¹   7.013k ± ∞ ¹        ~ (p=1.000 n=1) ²
geomean                                                 18.95k         11.64k        -38.56%
¹ need >= 6 samples for confidence interval at level 0.95
² need >= 4 samples to detect a difference at alpha level 0.05
```
  • Loading branch information
atoulme authored Jul 11, 2024
1 parent 9a3e1de commit 6ed999b
Show file tree
Hide file tree
Showing 10 changed files with 62 additions and 48 deletions.
27 changes: 27 additions & 0 deletions .chloggen/hec_json.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Use this changelog template to create an entry for release notes.

# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
change_type: enhancement

# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
component: splunkhecexporter

# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
note: Increase the performance of JSON marshaling

# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
issues: [34011]

# (Optional) One or more lines of additional information to render under the primary note.
# These lines will be padded with 2 spaces and then inserted directly into the document.
# Use pipe (|) for multiline entries.
subtext:

# If your change doesn't affect end users or the exported elements of any package,
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
# Optional: The change log or logs in which this entry should be included.
# e.g. '[user]' or '[user, api]'
# Include 'user' if the change is relevant to end users.
# Include 'api' if there is a change to a library API.
# Default: '[user]'
change_logs: []
42 changes: 13 additions & 29 deletions exporter/splunkhecexporter/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
"net/url"
"sync"

jsoniter "github.com/json-iterator/go"
"github.com/goccy/go-json"
"go.opentelemetry.io/collector/component"
"go.opentelemetry.io/collector/consumer/consumererror"
"go.opentelemetry.io/collector/exporter"
Expand Down Expand Up @@ -58,12 +58,6 @@ type client struct {
meter metric.Meter
}

var jsonStreamPool = sync.Pool{
New: func() any {
return jsoniter.NewStream(jsoniter.ConfigDefault, nil, 512)
},
}

func newClient(set exporter.Settings, cfg *Config, maxContentLength uint) *client {
return &client{
config: cfg,
Expand Down Expand Up @@ -196,8 +190,6 @@ func (c *client) pushLogDataInBatches(ctx context.Context, ld plog.Logs, headers
func (c *client) fillLogsBuffer(logs plog.Logs, buf buffer, is iterState) (iterState, []error) {
var b []byte
var permanentErrors []error
jsonStream := jsonStreamPool.Get().(*jsoniter.Stream)
defer jsonStreamPool.Put(jsonStream)

for i := is.resource; i < logs.ResourceLogs().Len(); i++ {
rl := logs.ResourceLogs().At(i)
Expand All @@ -216,7 +208,7 @@ func (c *client) fillLogsBuffer(logs plog.Logs, buf buffer, is iterState) (iterS

// JSON encoding event and writing to buffer.
var err error
b, err = marshalEvent(event, c.config.MaxEventSize, jsonStream)
b, err = marshalEvent(event, c.config.MaxEventSize)
if err != nil {
permanentErrors = append(permanentErrors, consumererror.NewPermanent(fmt.Errorf(
"dropped log event: %v, error: %w", event, err)))
Expand Down Expand Up @@ -249,8 +241,6 @@ func (c *client) fillLogsBuffer(logs plog.Logs, buf buffer, is iterState) (iterS

func (c *client) fillMetricsBuffer(metrics pmetric.Metrics, buf buffer, is iterState) (iterState, []error) {
var permanentErrors []error
jsonStream := jsonStreamPool.Get().(*jsoniter.Stream)
defer jsonStreamPool.Put(jsonStream)

tempBuf := bytes.NewBuffer(make([]byte, 0, c.config.MaxContentLengthMetrics))
for i := is.resource; i < metrics.ResourceMetrics().Len(); i++ {
Expand All @@ -267,7 +257,7 @@ func (c *client) fillMetricsBuffer(metrics pmetric.Metrics, buf buffer, is iterS
tempBuf.Reset()
for _, event := range events {
// JSON encoding event and writing to buffer.
b, err := marshalEvent(event, c.config.MaxEventSize, jsonStream)
b, err := marshalEvent(event, c.config.MaxEventSize)
if err != nil {
permanentErrors = append(permanentErrors, consumererror.NewPermanent(fmt.Errorf("dropped metric event: %v, error: %w", event, err)))
continue
Expand Down Expand Up @@ -301,13 +291,11 @@ func (c *client) fillMetricsBuffer(metrics pmetric.Metrics, buf buffer, is iterS

func (c *client) fillMetricsBufferMultiMetrics(events []*splunk.Event, buf buffer, is iterState) (iterState, []error) {
var permanentErrors []error
jsonStream := jsonStreamPool.Get().(*jsoniter.Stream)
defer jsonStreamPool.Put(jsonStream)

for i := is.record; i < len(events); i++ {
event := events[i]
// JSON encoding event and writing to buffer.
b, jsonErr := marshalEvent(event, c.config.MaxEventSize, jsonStream)
b, jsonErr := marshalEvent(event, c.config.MaxEventSize)
if jsonErr != nil {
permanentErrors = append(permanentErrors, consumererror.NewPermanent(fmt.Errorf("dropped metric event: %v, error: %w", event, jsonErr)))
continue
Expand Down Expand Up @@ -338,8 +326,6 @@ func (c *client) fillMetricsBufferMultiMetrics(events []*splunk.Event, buf buffe

func (c *client) fillTracesBuffer(traces ptrace.Traces, buf buffer, is iterState) (iterState, []error) {
var permanentErrors []error
jsonStream := jsonStreamPool.Get().(*jsoniter.Stream)
defer jsonStreamPool.Put(jsonStream)

for i := is.resource; i < traces.ResourceSpans().Len(); i++ {
rs := traces.ResourceSpans().At(i)
Expand All @@ -354,7 +340,7 @@ func (c *client) fillTracesBuffer(traces ptrace.Traces, buf buffer, is iterState
event := mapSpanToSplunkEvent(rs.Resource(), span, c.config)

// JSON encoding event and writing to buffer.
b, err := marshalEvent(event, c.config.MaxEventSize, jsonStream)
b, err := marshalEvent(event, c.config.MaxEventSize)
if err != nil {
permanentErrors = append(permanentErrors, consumererror.NewPermanent(fmt.Errorf("dropped span events: %v, error: %w", event, err)))
continue
Expand Down Expand Up @@ -687,16 +673,14 @@ func buildHTTPHeaders(config *Config, buildInfo component.BuildInfo) map[string]
}
}

// marshalEvent marshals an event to JSON using a reusable jsoniter stream.
func marshalEvent(event *splunk.Event, sizeLimit uint, stream *jsoniter.Stream) ([]byte, error) {
stream.Reset(nil)
stream.Error = nil
stream.WriteVal(event)
if stream.Error != nil {
return nil, stream.Error
// marshalEvent marshals an event to JSON
func marshalEvent(event *splunk.Event, sizeLimit uint) ([]byte, error) {
b, err := json.Marshal(event)
if err != nil {
return nil, err
}
if uint(stream.Buffered()) > sizeLimit {
return nil, fmt.Errorf("event size %d exceeds limit %d", stream.Buffered(), sizeLimit)
if uint(len(b)) > sizeLimit {
return nil, fmt.Errorf("event size %d exceeds limit %d", len(b), sizeLimit)
}
return stream.Buffer(), nil
return b, nil
}
9 changes: 4 additions & 5 deletions exporter/splunkhecexporter/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (
"testing"
"time"

jsoniter "github.com/json-iterator/go"
"github.com/goccy/go-json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/collector/component"
Expand Down Expand Up @@ -978,16 +978,15 @@ func TestReceiveSpanEvent(t *testing.T) {
}

// compareWithTestData compares hec output with a json file using maps instead of strings to avoid key ordering
// issues (jsoniter doesn't sort the keys).
func compareWithTestData(t *testing.T, actual []byte, file string) {
wantStr, err := os.ReadFile(file)
require.NoError(t, err)
wantMap := map[string]any{}
err = jsoniter.Unmarshal(wantStr, &wantMap)
err = json.Unmarshal(wantStr, &wantMap)
require.NoError(t, err)

gotMap := map[string]any{}
err = jsoniter.Unmarshal(actual, &gotMap)
err = json.Unmarshal(actual, &gotMap)
require.NoError(t, err)
assert.Equal(t, wantMap, gotMap)
}
Expand Down Expand Up @@ -1481,7 +1480,7 @@ func TestInvalidJson(t *testing.T) {
badEvent := badJSON{
Foo: math.Inf(1),
}
_, err := jsoniter.Marshal(badEvent)
_, err := json.Marshal(badEvent)
assert.Error(t, err)
}

Expand Down
3 changes: 2 additions & 1 deletion exporter/splunkhecexporter/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.21.0

require (
github.com/cenkalti/backoff/v4 v4.3.0
github.com/json-iterator/go v1.1.12
github.com/goccy/go-json v0.9.7
github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.104.0
github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk v0.104.0
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/batchperresourceattr v0.104.0
Expand Down Expand Up @@ -57,6 +57,7 @@ require (
github.com/golang/snappy v0.0.4 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/hashicorp/go-version v1.7.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/knadh/koanf/maps v0.1.1 // indirect
github.com/knadh/koanf/providers/confmap v0.1.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions exporter/splunkhecexporter/go.sum

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 5 additions & 5 deletions exporter/splunkhecexporter/logdata_to_splunk.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"fmt"
"time"

jsoniter "github.com/json-iterator/go"
"github.com/goccy/go-json"
"go.opentelemetry.io/collector/pdata/pcommon"
"go.opentelemetry.io/collector/pdata/plog"

Expand Down Expand Up @@ -110,8 +110,8 @@ func mergeValue(dst map[string]any, k string, v any) {
if isArrayFlat(element) {
dst[k] = v
} else {
jsonStr, _ := jsoniter.MarshalToString(element)
dst[k] = jsonStr
b, _ := json.Marshal(element)
dst[k] = string(b)
}
case map[string]any:
flattenAndMergeMap(element, dst, k)
Expand Down Expand Up @@ -141,8 +141,8 @@ func flattenAndMergeMap(src, dst map[string]any, key string) {
if isArrayFlat(element) {
dst[current] = element
} else {
jsonStr, _ := jsoniter.MarshalToString(element)
dst[current] = jsonStr
b, _ := json.Marshal(element)
dst[current] = string(b)
}

default:
Expand Down
5 changes: 2 additions & 3 deletions exporter/splunkhecexporter/metricdata_to_splunk.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"strconv"
"strings"

jsoniter "github.com/json-iterator/go"
"github.com/goccy/go-json"
"go.opentelemetry.io/collector/pdata/pcommon"
"go.opentelemetry.io/collector/pdata/pmetric"
"go.uber.org/zap"
Expand Down Expand Up @@ -284,12 +284,11 @@ func mergeEventsToMultiMetricFormat(events []*splunk.Event) ([]*splunk.Event, er
hashes := map[uint32]*splunk.Event{}
hasher := fnv.New32a()
var merged []*splunk.Event
marshaler := jsoniter.ConfigCompatibleWithStandardLibrary

for _, e := range events {
cloned := copyEventWithoutValues(e)

data, err := marshaler.Marshal(cloned)
data, err := json.Marshal(cloned)
if err != nil {
return nil, err
}
Expand Down
9 changes: 4 additions & 5 deletions exporter/splunkhecexporter/metricdata_to_splunk_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,13 @@
package splunkhecexporter

import (
"encoding/json"
"fmt"
"io"
"math"
"testing"
"time"

jsoniter "github.com/json-iterator/go"
"github.com/goccy/go-json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/collector/pdata/pcommon"
Expand Down Expand Up @@ -706,16 +705,16 @@ func TestMergeEvents(t *testing.T) {
json1 := `{"event":"metric","fields":{"IF-Azure":"azure-env","k8s.cluster.name":"devops-uat","k8s.namespace.name":"splunk-collector-tests","k8s.node.name":"myk8snodename","k8s.pod.name":"my-otel-collector-pod","metric_type":"Gauge","metricsIndex":"test_metrics","metricsPlatform":"unset","resourceAttrs":"NO","testNumber":"number42","testRun":"42","metric_name:otel.collector.test":3411}}`
json2 := `{"event":"metric","fields":{"IF-Azure":"azure-env","k8s.cluster.name":"devops-uat","k8s.namespace.name":"splunk-collector-tests","k8s.node.name":"myk8snodename","k8s.pod.name":"my-otel-collector-pod","metric_type":"Gauge","metricsIndex":"test_metrics","metricsPlatform":"unset","resourceAttrs":"NO","testNumber":"number42","testRun":"42","metric_name:otel.collector.test2":26059}}`
ev1 := &splunk.Event{}
err := jsoniter.Unmarshal([]byte(json1), ev1)
err := json.Unmarshal([]byte(json1), ev1)
require.NoError(t, err)
ev2 := &splunk.Event{}
err = jsoniter.Unmarshal([]byte(json2), ev2)
err = json.Unmarshal([]byte(json2), ev2)
require.NoError(t, err)
events := []*splunk.Event{ev1, ev2}
merged, err := mergeEventsToMultiMetricFormat(events)
require.NoError(t, err)
require.Len(t, merged, 1)
b, err := jsoniter.ConfigCompatibleWithStandardLibrary.Marshal(merged[0])
b, err := json.Marshal(merged[0])
require.NoError(t, err)
require.Equal(t, `{"host":"","event":"metric","fields":{"IF-Azure":"azure-env","k8s.cluster.name":"devops-uat","k8s.namespace.name":"splunk-collector-tests","k8s.node.name":"myk8snodename","k8s.pod.name":"my-otel-collector-pod","metric_name:otel.collector.test":3411,"metric_name:otel.collector.test2":26059,"metric_type":"Gauge","metricsIndex":"test_metrics","metricsPlatform":"unset","resourceAttrs":"NO","testNumber":"number42","testRun":"42"}}`, string(b))
}
Expand Down
1 change: 1 addition & 0 deletions receiver/splunkhecreceiver/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ require (
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect
github.com/goccy/go-json v0.9.7 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/hashicorp/go-version v1.7.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions receiver/splunkhecreceiver/go.sum

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 6ed999b

Please sign in to comment.