-
Notifications
You must be signed in to change notification settings - Fork 1.2k
/
Copy pathpkg_metrics.yml
137 lines (122 loc) · 7.63 KB
/
pkg_metrics.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
---
# pkg_metrics stage
# Contains jobs which send metrics (package size) about the Linus Agent packages to our backend.
send_pkg_size-a6:
allow_failure: true
rules:
!reference [.on_deploy_a6]
stage: pkg_metrics
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64:$DATADOG_AGENT_BUILDIMAGES
tags: ["runner:main"]
dependencies:
- agent_deb-x64-a6
- agent_rpm-x64-a6
- agent_suse-x64-a6
before_script:
# FIXME: tmp while we uppdate the base image
- apt-get install -y wget rpm2cpio cpio
- ls -l $OMNIBUS_PACKAGE_DIR
- ls -l $OMNIBUS_PACKAGE_DIR_SUSE
script:
- source /root/.bashrc && conda activate ddpy3
- mkdir -p /tmp/deb/agent
- mkdir -p /tmp/rpm/agent
- mkdir -p /tmp/suse/agent
# we silence dpkg and cpio output so we don't exceed gitlab log limit
# debian
- dpkg -x $OMNIBUS_PACKAGE_DIR/datadog-agent_6*_amd64.deb /tmp/deb/agent > /dev/null
- DEB_AGENT_SIZE=$(du -sB1 /tmp/deb/agent | sed 's/\([0-9]\+\).\+/\1/')
# centos
- cd /tmp/rpm/agent && rpm2cpio $OMNIBUS_PACKAGE_DIR/datadog-agent-6.*.x86_64.rpm | cpio -idm > /dev/null
- RPM_AGENT_SIZE=$(du -sB1 /tmp/rpm/agent | sed 's/\([0-9]\+\).\+/\1/')
# suse
- cd /tmp/suse/agent && rpm2cpio $OMNIBUS_PACKAGE_DIR_SUSE/datadog-agent-6.*.x86_64.rpm | cpio -idm > /dev/null
- SUSE_AGENT_SIZE=$(du -sB1 /tmp/suse/agent | sed 's/\([0-9]\+\).\+/\1/')
- currenttime=$(date +%s)
- DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key --with-decryption --query "Parameter.Value" --out text)
- |
curl --fail -X POST -H "Content-type: application/json" \
-d "{\"series\":[
{\"metric\":\"datadog.agent.package.size\",\"points\":[[$currenttime, $DEB_AGENT_SIZE]], \"tags\":[\"os:debian\", \"package:agent\", \"agent:6\", \"bucket_branch:$DEB_RPM_BUCKET_BRANCH\"]},
{\"metric\":\"datadog.agent.package.size\",\"points\":[[$currenttime, $RPM_AGENT_SIZE]], \"tags\":[\"os:centos\", \"package:agent\", \"agent:6\", \"bucket_branch:$DEB_RPM_BUCKET_BRANCH\"]},
{\"metric\":\"datadog.agent.package.size\",\"points\":[[$currenttime, $SUSE_AGENT_SIZE]], \"tags\":[\"os:suse\", \"package:agent\", \"agent:6\", \"bucket_branch:$DEB_RPM_BUCKET_BRANCH\"]}
]}" \
"https://api.datadoghq.com/api/v1/series?api_key=$DD_API_KEY"
send_pkg_size-a7:
allow_failure: true
rules:
!reference [.on_deploy_a7]
stage: pkg_metrics
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64:$DATADOG_AGENT_BUILDIMAGES
tags: ["runner:main"]
dependencies:
- agent_deb-x64-a7
- iot_agent_deb-x64
- dogstatsd_deb-x64
- agent_rpm-x64-a7
- iot_agent_rpm-x64
- dogstatsd_rpm-x64
- agent_suse-x64-a7
- dogstatsd_suse-x64
- iot_agent_suse-x64
before_script:
# FIXME: tmp while we uppdate the base image
- apt-get install -y wget rpm2cpio cpio jq
- ls -l $OMNIBUS_PACKAGE_DIR
- ls -l $OMNIBUS_PACKAGE_DIR_SUSE
script:
- source /root/.bashrc && conda activate ddpy3
- mkdir -p /tmp/deb/agent /tmp/deb/dogstatsd /tmp/deb/iot-agent
- mkdir -p /tmp/rpm/agent /tmp/rpm/dogstatsd /tmp/rpm/iot-agent
- mkdir -p /tmp/suse/agent /tmp/suse/dogstatsd /tmp/suse/iot-agent
# we will build up a JSON body in `$metrics`, and then post it.
- 'post_body="{\"series\": []}"'
- |
add_metric() {
local metric="${1}"
shift
local value="${1}"
shift
local tags=[]
while [ -n "${1}" ]; do
tags=$(echo $tags | jq -c ". += [\"${1}\"]")
shift
done
post_body=$(echo $post_body | jq -c ".series += [{\"metric\":\"$metric\", \"points\":[[$currenttime, $value]],\"tags\":$tags}]")
}
- |
add_metrics() {
local base="${1}"
local os="${2}"
# record a the total uncompressed size of each package
add_metric datadog.agent.package.size $(du -sB1 ${base}/agent | sed 's/\([0-9]\+\).\+/\1/') os:${os} package:agent agent:7 bucket_branch:$DEB_RPM_BUCKET_BRANCH
add_metric datadog.agent.package.size $(du -sB1 ${base}/dogstatsd | sed 's/\([0-9]\+\).\+/\1/') os:${os} package:dogstatsd agent:7 bucket_branch:$DEB_RPM_BUCKET_BRANCH
add_metric datadog.agent.package.size $(du -sB1 ${base}/iot-agent | sed 's/\([0-9]\+\).\+/\1/') os:${os} package:iot-agent agent:7 bucket_branch:$DEB_RPM_BUCKET_BRANCH
# record the size of each of the important binaries in each package
add_metric datadog.agent.binary.size $(du -sB1 ${base}/agent/opt/datadog-agent/bin/agent/agent | sed 's/\([0-9]\+\).\+/\1/') bin:agent os:${os} package:agent agent:7 bucket_branch:$DEB_RPM_BUCKET_BRANCH
add_metric datadog.agent.binary.size $(du -sB1 ${base}/agent/opt/datadog-agent/embedded/bin/trace-agent | sed 's/\([0-9]\+\).\+/\1/') bin:trace-agent os:${os} package:agent agent:7 bucket_branch:$DEB_RPM_BUCKET_BRANCH
add_metric datadog.agent.binary.size $(du -sB1 ${base}/agent/opt/datadog-agent/embedded/bin/security-agent | sed 's/\([0-9]\+\).\+/\1/') bin:security-agent os:${os} package:agent agent:7 bucket_branch:$DEB_RPM_BUCKET_BRANCH
add_metric datadog.agent.binary.size $(du -sB1 ${base}/agent/opt/datadog-agent/embedded/bin/process-agent | sed 's/\([0-9]\+\).\+/\1/') bin:process-agent os:${os} package:agent agent:7 bucket_branch:$DEB_RPM_BUCKET_BRANCH
add_metric datadog.agent.binary.size $(du -sB1 ${base}/agent/opt/datadog-agent/embedded/bin/system-probe | sed 's/\([0-9]\+\).\+/\1/') bin:system-probe os:${os} package:agent agent:7 bucket_branch:$DEB_RPM_BUCKET_BRANCH
add_metric datadog.agent.binary.size $(du -sB1 ${base}/dogstatsd/opt/datadog-dogstatsd/bin/dogstatsd | sed 's/\([0-9]\+\).\+/\1/') bin:dogstatsd os:${os} package:dogstatsd agent:7 bucket_branch:$DEB_RPM_BUCKET_BRANCH
add_metric datadog.agent.binary.size $(du -sB1 ${base}/iot-agent/opt/datadog-agent/bin/agent/agent | sed 's/\([0-9]\+\).\+/\1/') bin:agent os:${os} package:iot-agent agent:7 bucket_branch:$DEB_RPM_BUCKET_BRANCH
}
- currenttime=$(date +%s)
# We silence dpkg and cpio output so we don't exceed gitlab log limit
# debian
- dpkg -x $OMNIBUS_PACKAGE_DIR/datadog-agent_7*_amd64.deb /tmp/deb/agent > /dev/null
- dpkg -x $OMNIBUS_PACKAGE_DIR/datadog-iot-agent_7*_amd64.deb /tmp/deb/iot-agent > /dev/null
- dpkg -x $OMNIBUS_PACKAGE_DIR/datadog-dogstatsd_7*_amd64.deb /tmp/deb/dogstatsd > /dev/null
- add_metrics /tmp/deb debian
# centos
- cd /tmp/rpm/agent && rpm2cpio $OMNIBUS_PACKAGE_DIR/datadog-agent-7.*.x86_64.rpm | cpio -idm > /dev/null
- cd /tmp/rpm/dogstatsd && rpm2cpio $OMNIBUS_PACKAGE_DIR/datadog-dogstatsd-7.*.x86_64.rpm | cpio -idm > /dev/null
- cd /tmp/rpm/iot-agent && rpm2cpio $OMNIBUS_PACKAGE_DIR/datadog-iot-agent-7.*.x86_64.rpm | cpio -idm > /dev/null
- add_metrics /tmp/rpm centos
# suse
- cd /tmp/suse/agent && rpm2cpio $OMNIBUS_PACKAGE_DIR_SUSE/datadog-agent-7.*.x86_64.rpm | cpio -idm > /dev/null
- cd /tmp/suse/dogstatsd && rpm2cpio $OMNIBUS_PACKAGE_DIR_SUSE/datadog-dogstatsd-7.*.x86_64.rpm | cpio -idm > /dev/null
- cd /tmp/suse/iot-agent && rpm2cpio $OMNIBUS_PACKAGE_DIR_SUSE/datadog-iot-agent-7.*.x86_64.rpm | cpio -idm > /dev/null
- add_metrics /tmp/suse suse
- DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key --with-decryption --query "Parameter.Value" --out text)
- 'curl --fail -X POST -H "Content-type: application/json" -d "$post_body" "https://api.datadoghq.com/api/v1/series?api_key=$DD_API_KEY"'