Skip to content

Commit 702328e

Browse files
fargitofargito
committed
feat: add run_part to upload metadata
1 parent 0f0cce3 commit 702328e

16 files changed

+489
-11
lines changed

‎src/run/ci_provider/buildkite/provider.rs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ use std::env;
33
use simplelog::SharedLogger;
44

55
use crate::prelude::*;
6-
use crate::run::ci_provider::interfaces::Platform;
6+
use crate::run::ci_provider::interfaces::{Platform, RunPart};
77
use crate::run::helpers::{parse_git_remote, GitRemote};
88
use crate::run::{
99
ci_provider::{
@@ -132,6 +132,11 @@ impl CIProvider for BuildkiteProvider {
132132
Platform::Buildkite
133133
}
134134

135+
/// For Buildkite, we don't support multipart uploads
136+
fn get_platform_run_part(&self) -> Option<RunPart> {
137+
None
138+
}
139+
135140
fn get_ci_provider_metadata(&self) -> Result<CIProviderMetadata> {
136141
Ok(CIProviderMetadata {
137142
base_ref: self.base_ref.clone(),

‎src/run/ci_provider/github_actions/provider.rs‎

Lines changed: 308 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@ use lazy_static::lazy_static;
22
use regex::Regex;
33
use serde_json::Value;
44
use simplelog::SharedLogger;
5+
use std::collections::BTreeMap;
56
use std::{env, fs};
67

78
use crate::prelude::*;
8-
use crate::run::ci_provider::interfaces::Platform;
9+
use crate::run::ci_provider::interfaces::{Platform, RunPart};
910
use crate::run::{
1011
ci_provider::{
1112
interfaces::{CIProviderMetadata, GhData, RepositoryProvider, RunEvent, Sender},
@@ -133,6 +134,64 @@ impl CIProvider for GitHubActionsProvider {
133134
Platform::GithubActions
134135
}
135136

137+
/// For Github, the platform run part is the most complicated
138+
/// since we support matrix jobs.
139+
///
140+
/// Computing the `run_part_id`:
141+
/// - not in a matrix:
142+
/// - simply take the job name
143+
/// - in a matrix:
144+
/// - take the job name
145+
/// - concatenate it with key-values from `matrix` and `strategy`
146+
///
147+
/// `GH_MATRIX` and `GH_STRATEGY` are environment variables computed by
148+
/// https://github.com/CodSpeedHQ/action:
149+
/// - `GH_MATRIX`: ${{ toJson(matrix) }}
150+
/// - `GH_STRATEGY`: ${{ toJson(strategy) }}
151+
///
152+
/// A note on parsing:
153+
///
154+
/// The issue is these variables from Github Actions are multiline.
155+
/// As we need to use them compute an identifier, we need them as a single line.
156+
/// Plus we are interested in the content of these objects,
157+
/// so it makes sense to parse and re-serialize them.
158+
fn get_platform_run_part(&self) -> Option<RunPart> {
159+
let job_name = self.gh_data.job.clone();
160+
161+
let mut metadata = BTreeMap::new();
162+
163+
let gh_matrix = get_env_variable("GH_MATRIX")
164+
.ok()
165+
.and_then(|v| serde_json::from_str::<Value>(&v).ok());
166+
167+
let gh_strategy = get_env_variable("GH_STRATEGY")
168+
.ok()
169+
.and_then(|v| serde_json::from_str::<Value>(&v).ok());
170+
171+
let run_part_id = if let (Some(Value::Object(matrix)), Some(Value::Object(strategy))) =
172+
(gh_matrix, gh_strategy)
173+
{
174+
// The re-serialization is on purpose here. We want to serialize it as a single line.
175+
let matrix_str = serde_json::to_string(&matrix).expect("Unable to re-serialize matrix");
176+
let strategy_str =
177+
serde_json::to_string(&strategy).expect("Unable to re-serialize strategy");
178+
179+
metadata.extend(matrix);
180+
metadata.extend(strategy);
181+
182+
format!("{job_name}-{matrix_str}-{strategy_str}")
183+
} else {
184+
job_name
185+
};
186+
187+
Some(RunPart {
188+
run_id: self.gh_data.run_id.clone(),
189+
run_part_id,
190+
job_name: self.gh_data.job.clone(),
191+
metadata,
192+
})
193+
}
194+
136195
fn get_ci_provider_metadata(&self) -> Result<CIProviderMetadata> {
137196
Ok(CIProviderMetadata {
138197
base_ref: self.base_ref.clone(),
@@ -246,13 +305,15 @@ mod tests {
246305
};
247306
let github_actions_provider = GitHubActionsProvider::try_from(&config).unwrap();
248307
let provider_metadata = github_actions_provider.get_ci_provider_metadata().unwrap();
308+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
249309

250310
assert_json_snapshot!(provider_metadata, {
251311
".runner.version" => insta::dynamic_redaction(|value,_path| {
252312
assert_eq!(value.as_str().unwrap(), VERSION.to_string());
253313
"[version]"
254314
}),
255315
});
316+
assert_json_snapshot!(run_part);
256317
},
257318
);
258319
}
@@ -282,6 +343,7 @@ mod tests {
282343
("GITHUB_REPOSITORY", Some("my-org/adrien-python-test")),
283344
("GITHUB_RUN_ID", Some("6957110437")),
284345
("VERSION", Some("0.1.0")),
346+
("GH_MATRIX", Some("null")),
285347
],
286348
|| {
287349
let config = Config {
@@ -290,6 +352,7 @@ mod tests {
290352
};
291353
let github_actions_provider = GitHubActionsProvider::try_from(&config).unwrap();
292354
let provider_metadata = github_actions_provider.get_ci_provider_metadata().unwrap();
355+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
293356

294357
assert_eq!(provider_metadata.owner, "my-org");
295358
assert_eq!(provider_metadata.repository, "adrien-python-test");
@@ -298,13 +361,257 @@ mod tests {
298361
provider_metadata.head_ref,
299362
Some("fork-owner:feat/codspeed-runner".into())
300363
);
364+
365+
assert_json_snapshot!(provider_metadata, {
366+
".runner.version" => insta::dynamic_redaction(|value,_path| {
367+
assert_eq!(value.as_str().unwrap(), VERSION.to_string());
368+
"[version]"
369+
}),
370+
});
371+
assert_json_snapshot!(run_part);
372+
},
373+
);
374+
}
375+
376+
#[test]
377+
fn test_matrix_job_provider_metadata() {
378+
with_vars(
379+
[
380+
("GITHUB_ACTIONS", Some("true")),
381+
("GITHUB_ACTOR_ID", Some("19605940")),
382+
("GITHUB_ACTOR", Some("adriencaccia")),
383+
("GITHUB_BASE_REF", Some("main")),
384+
("GITHUB_EVENT_NAME", Some("pull_request")),
385+
(
386+
"GITHUB_EVENT_PATH",
387+
Some(
388+
format!(
389+
"{}/src/run/ci_provider/github_actions/samples/pr-event.json",
390+
env!("CARGO_MANIFEST_DIR")
391+
)
392+
.as_str(),
393+
),
394+
),
395+
("GITHUB_HEAD_REF", Some("feat/codspeed-runner")),
396+
("GITHUB_JOB", Some("log-env")),
397+
("GITHUB_REF", Some("refs/pull/22/merge")),
398+
("GITHUB_REPOSITORY", Some("my-org/adrien-python-test")),
399+
("GITHUB_RUN_ID", Some("6957110437")),
400+
("VERSION", Some("0.1.0")),
401+
(
402+
"GH_MATRIX",
403+
Some(
404+
r#"{
405+
"runner-version":"3.2.1",
406+
"numeric-value":123456789
407+
}"#,
408+
),
409+
),
410+
(
411+
"GH_STRATEGY",
412+
Some(
413+
r#"{
414+
"fail-fast":true,
415+
"job-index":1,
416+
"job-total":2,
417+
"max-parallel":2
418+
}"#,
419+
),
420+
),
421+
],
422+
|| {
423+
let config = Config {
424+
token: Some("token".into()),
425+
..Config::test()
426+
};
427+
let github_actions_provider = GitHubActionsProvider::try_from(&config).unwrap();
428+
let provider_metadata = github_actions_provider.get_ci_provider_metadata().unwrap();
429+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
430+
301431
assert_json_snapshot!(provider_metadata, {
302432
".runner.version" => insta::dynamic_redaction(|value,_path| {
303433
assert_eq!(value.as_str().unwrap(), VERSION.to_string());
304434
"[version]"
305435
}),
306436
});
437+
assert_json_snapshot!(run_part);
307438
},
308439
);
309440
}
441+
442+
#[test]
443+
fn test_get_run_part_no_matrix() {
444+
let github_actions_provider = GitHubActionsProvider {
445+
owner: "owner".into(),
446+
repository: "repository".into(),
447+
ref_: "refs/head/my-branch".into(),
448+
head_ref: Some("my-branch".into()),
449+
base_ref: None,
450+
sender: None,
451+
gh_data: GhData {
452+
job: "my_job".into(),
453+
run_id: "123789".into(),
454+
},
455+
event: RunEvent::Push,
456+
repository_root_path: "/home/work/my-repo".into(),
457+
};
458+
459+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
460+
461+
assert_eq!(run_part.run_id, "123789");
462+
assert_eq!(run_part.job_name, "my_job");
463+
assert_eq!(run_part.run_part_id, "my_job");
464+
assert_json_snapshot!(run_part.metadata, @"{}");
465+
}
466+
467+
#[test]
468+
fn test_get_run_part_null_matrix() {
469+
with_vars(
470+
[
471+
("GH_MATRIX", Some("null")),
472+
(
473+
"GH_STRATEGY",
474+
Some(
475+
r#"{
476+
"fail-fast":true,
477+
"job-index":0,
478+
"job-total":1,
479+
"max-parallel":1
480+
}"#,
481+
),
482+
),
483+
],
484+
|| {
485+
let github_actions_provider = GitHubActionsProvider {
486+
owner: "owner".into(),
487+
repository: "repository".into(),
488+
ref_: "refs/head/my-branch".into(),
489+
head_ref: Some("my-branch".into()),
490+
base_ref: None,
491+
sender: None,
492+
gh_data: GhData {
493+
job: "my_job".into(),
494+
run_id: "123789".into(),
495+
},
496+
event: RunEvent::Push,
497+
repository_root_path: "/home/work/my-repo".into(),
498+
};
499+
500+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
501+
502+
assert_eq!(run_part.run_id, "123789");
503+
assert_eq!(run_part.job_name, "my_job");
504+
assert_eq!(run_part.run_part_id, "my_job");
505+
assert_json_snapshot!(run_part.metadata, @"{}");
506+
},
507+
)
508+
}
509+
510+
#[test]
511+
fn test_get_matrix_run_part() {
512+
with_vars(
513+
[
514+
(
515+
"GH_MATRIX",
516+
Some(
517+
r#"{
518+
"runner-version":"3.2.1",
519+
"numeric-value":123456789
520+
}"#,
521+
),
522+
),
523+
(
524+
"GH_STRATEGY",
525+
Some(
526+
r#"{
527+
"fail-fast":true,
528+
"job-index":1,
529+
"job-total":2,
530+
"max-parallel":2
531+
}"#,
532+
),
533+
),
534+
],
535+
|| {
536+
let github_actions_provider = GitHubActionsProvider {
537+
owner: "owner".into(),
538+
repository: "repository".into(),
539+
ref_: "refs/head/my-branch".into(),
540+
head_ref: Some("my-branch".into()),
541+
base_ref: None,
542+
sender: None,
543+
gh_data: GhData {
544+
job: "my_job".into(),
545+
run_id: "123789".into(),
546+
},
547+
event: RunEvent::Push,
548+
repository_root_path: "/home/work/my-repo".into(),
549+
};
550+
551+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
552+
553+
assert_eq!(run_part.run_id, "123789");
554+
assert_eq!(run_part.job_name, "my_job");
555+
assert_eq!(run_part.run_part_id, "my_job-{\"runner-version\":\"3.2.1\",\"numeric-value\":123456789}-{\"fail-fast\":true,\"job-index\":1,\"job-total\":2,\"max-parallel\":2}");
556+
assert_json_snapshot!(run_part.metadata, @r#"
557+
{
558+
"fail-fast": true,
559+
"job-index": 1,
560+
"job-total": 2,
561+
"max-parallel": 2,
562+
"numeric-value": 123456789,
563+
"runner-version": "3.2.1"
564+
}
565+
"#);
566+
},
567+
)
568+
}
569+
570+
#[test]
571+
fn test_get_inline_matrix_run_part() {
572+
with_vars(
573+
[
574+
(
575+
"GH_MATRIX",
576+
Some("{\"runner-version\":\"3.2.1\",\"numeric-value\":123456789}"),
577+
),
578+
(
579+
"GH_STRATEGY",
580+
Some("{\"fail-fast\":true,\"job-index\":1,\"job-total\":2,\"max-parallel\":2}"),
581+
),
582+
],
583+
|| {
584+
let github_actions_provider = GitHubActionsProvider {
585+
owner: "owner".into(),
586+
repository: "repository".into(),
587+
ref_: "refs/head/my-branch".into(),
588+
head_ref: Some("my-branch".into()),
589+
base_ref: None,
590+
sender: None,
591+
gh_data: GhData {
592+
job: "my_job".into(),
593+
run_id: "123789".into(),
594+
},
595+
event: RunEvent::Push,
596+
repository_root_path: "/home/work/my-repo".into(),
597+
};
598+
599+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
600+
601+
assert_eq!(run_part.run_id, "123789");
602+
assert_eq!(run_part.job_name, "my_job");
603+
assert_eq!(run_part.run_part_id, "my_job-{\"runner-version\":\"3.2.1\",\"numeric-value\":123456789}-{\"fail-fast\":true,\"job-index\":1,\"job-total\":2,\"max-parallel\":2}");
604+
assert_json_snapshot!(run_part.metadata, @r#"
605+
{
606+
"fail-fast": true,
607+
"job-index": 1,
608+
"job-total": 2,
609+
"max-parallel": 2,
610+
"numeric-value": 123456789,
611+
"runner-version": "3.2.1"
612+
}
613+
"#);
614+
},
615+
)
616+
}
310617
}

‎src/run/ci_provider/github_actions/snapshots/codspeed__run__ci_provider__github_actions__provider__tests__fork_pull_request_provider_metadata-2.snap‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
source: src/run/ci_provider/github_actions/provider.rs
3+
expression: run_part
4+
---
5+
{
6+
"runId": "6957110437",
7+
"runPartId": "log-env",
8+
"jobName": "log-env",
9+
"metadata": {}
10+
}

0 commit comments

Comments
 (0)