Skip to content

Commit ed81950

Browse files
fargitofargito
committed
feat: add run_part to upload metadata
1 parent 0f0cce3 commit ed81950

16 files changed

+491
-11
lines changed

‎src/run/ci_provider/buildkite/provider.rs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ use std::env;
33
use simplelog::SharedLogger;
44

55
use crate::prelude::*;
6-
use crate::run::ci_provider::interfaces::Platform;
6+
use crate::run::ci_provider::interfaces::{Platform, RunPart};
77
use crate::run::helpers::{parse_git_remote, GitRemote};
88
use crate::run::{
99
ci_provider::{
@@ -132,6 +132,11 @@ impl CIProvider for BuildkiteProvider {
132132
Platform::Buildkite
133133
}
134134

135+
/// For Buildkite, we don't support multipart uploads
136+
fn get_platform_run_part(&self) -> Option<RunPart> {
137+
None
138+
}
139+
135140
fn get_ci_provider_metadata(&self) -> Result<CIProviderMetadata> {
136141
Ok(CIProviderMetadata {
137142
base_ref: self.base_ref.clone(),

‎src/run/ci_provider/github_actions/provider.rs‎

Lines changed: 310 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@ use lazy_static::lazy_static;
22
use regex::Regex;
33
use serde_json::Value;
44
use simplelog::SharedLogger;
5+
use std::collections::BTreeMap;
56
use std::{env, fs};
67

78
use crate::prelude::*;
8-
use crate::run::ci_provider::interfaces::Platform;
9+
use crate::run::ci_provider::interfaces::{Platform, RunPart};
910
use crate::run::{
1011
ci_provider::{
1112
interfaces::{CIProviderMetadata, GhData, RepositoryProvider, RunEvent, Sender},
@@ -133,6 +134,64 @@ impl CIProvider for GitHubActionsProvider {
133134
Platform::GithubActions
134135
}
135136

137+
/// For Github, the platform run part is the most complicated
138+
/// since we support matrix jobs.
139+
///
140+
/// Computing the `run_part_id`:
141+
/// - not in a matrix:
142+
/// - simply take the job name
143+
/// - in a matrix:
144+
/// - take the job name
145+
/// - concatenate it with key-values from `matrix` and `strategy`
146+
///
147+
/// `GH_MATRIX` and `GH_STRATEGY` are environment variables computed by
148+
/// https://github.com/CodSpeedHQ/action:
149+
/// - `GH_MATRIX`: ${{ toJson(matrix) }}
150+
/// - `GH_STRATEGY`: ${{ toJson(strategy) }}
151+
///
152+
/// A note on parsing:
153+
///
154+
/// The issue is these variables from Github Actions are multiline.
155+
/// As we need to use them compute an identifier, we need them as a single line.
156+
/// Plus we are interested in the content of these objects,
157+
/// so it makes sense to parse and re-serialize them.
158+
fn get_platform_run_part(&self) -> Option<RunPart> {
159+
let job_name = self.gh_data.job.clone();
160+
161+
let mut metadata = BTreeMap::new();
162+
163+
let gh_matrix = get_env_variable("GH_MATRIX")
164+
.ok()
165+
.and_then(|v| serde_json::from_str::<Value>(&v).ok());
166+
167+
let gh_strategy = get_env_variable("GH_STRATEGY")
168+
.ok()
169+
.and_then(|v| serde_json::from_str::<Value>(&v).ok());
170+
171+
let run_part_id = if let (Some(Value::Object(matrix)), Some(Value::Object(strategy))) =
172+
(gh_matrix, gh_strategy)
173+
{
174+
// The re-serialization is on purpose here. We want to serialize it as a single line.
175+
let matrix_str = serde_json::to_string(&matrix).expect("Unable to re-serialize matrix");
176+
let strategy_str =
177+
serde_json::to_string(&strategy).expect("Unable to re-serialize strategy");
178+
179+
metadata.extend(matrix);
180+
metadata.extend(strategy);
181+
182+
format!("{job_name}-{matrix_str}-{strategy_str}")
183+
} else {
184+
job_name
185+
};
186+
187+
Some(RunPart {
188+
run_id: self.gh_data.run_id.clone(),
189+
run_part_id,
190+
job_name: self.gh_data.job.clone(),
191+
metadata,
192+
})
193+
}
194+
136195
fn get_ci_provider_metadata(&self) -> Result<CIProviderMetadata> {
137196
Ok(CIProviderMetadata {
138197
base_ref: self.base_ref.clone(),
@@ -246,13 +305,15 @@ mod tests {
246305
};
247306
let github_actions_provider = GitHubActionsProvider::try_from(&config).unwrap();
248307
let provider_metadata = github_actions_provider.get_ci_provider_metadata().unwrap();
308+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
249309

250310
assert_json_snapshot!(provider_metadata, {
251311
".runner.version" => insta::dynamic_redaction(|value,_path| {
252312
assert_eq!(value.as_str().unwrap(), VERSION.to_string());
253313
"[version]"
254314
}),
255315
});
316+
assert_json_snapshot!(run_part);
256317
},
257318
);
258319
}
@@ -282,6 +343,7 @@ mod tests {
282343
("GITHUB_REPOSITORY", Some("my-org/adrien-python-test")),
283344
("GITHUB_RUN_ID", Some("6957110437")),
284345
("VERSION", Some("0.1.0")),
346+
("GH_MATRIX", Some("null")),
285347
],
286348
|| {
287349
let config = Config {
@@ -290,6 +352,7 @@ mod tests {
290352
};
291353
let github_actions_provider = GitHubActionsProvider::try_from(&config).unwrap();
292354
let provider_metadata = github_actions_provider.get_ci_provider_metadata().unwrap();
355+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
293356

294357
assert_eq!(provider_metadata.owner, "my-org");
295358
assert_eq!(provider_metadata.repository, "adrien-python-test");
@@ -298,13 +361,259 @@ mod tests {
298361
provider_metadata.head_ref,
299362
Some("fork-owner:feat/codspeed-runner".into())
300363
);
364+
301365
assert_json_snapshot!(provider_metadata, {
302366
".runner.version" => insta::dynamic_redaction(|value,_path| {
303367
assert_eq!(value.as_str().unwrap(), VERSION.to_string());
304368
"[version]"
305369
}),
306370
});
371+
assert_json_snapshot!(run_part);
307372
},
308373
);
309374
}
375+
376+
#[test]
377+
fn test_matrix_job_provider_metadata() {
378+
with_vars(
379+
[
380+
("GITHUB_ACTIONS", Some("true")),
381+
("GITHUB_ACTOR_ID", Some("19605940")),
382+
("GITHUB_ACTOR", Some("adriencaccia")),
383+
("GITHUB_BASE_REF", Some("main")),
384+
("GITHUB_EVENT_NAME", Some("pull_request")),
385+
(
386+
"GITHUB_EVENT_PATH",
387+
Some(
388+
format!(
389+
"{}/src/run/ci_provider/github_actions/samples/pr-event.json",
390+
env!("CARGO_MANIFEST_DIR")
391+
)
392+
.as_str(),
393+
),
394+
),
395+
("GITHUB_HEAD_REF", Some("feat/codspeed-runner")),
396+
("GITHUB_JOB", Some("log-env")),
397+
("GITHUB_REF", Some("refs/pull/22/merge")),
398+
("GITHUB_REPOSITORY", Some("my-org/adrien-python-test")),
399+
("GITHUB_RUN_ID", Some("6957110437")),
400+
("VERSION", Some("0.1.0")),
401+
(
402+
"GH_MATRIX",
403+
Some(
404+
r#"{
405+
"runner-version":"3.2.1",
406+
"numeric-value":123456789
407+
}"#,
408+
),
409+
),
410+
(
411+
"GH_STRATEGY",
412+
Some(
413+
r#"{
414+
"fail-fast":true,
415+
"job-index":1,
416+
"job-total":2,
417+
"max-parallel":2
418+
}"#,
419+
),
420+
),
421+
],
422+
|| {
423+
let config = Config {
424+
token: Some("token".into()),
425+
..Config::test()
426+
};
427+
let github_actions_provider = GitHubActionsProvider::try_from(&config).unwrap();
428+
let provider_metadata = github_actions_provider.get_ci_provider_metadata().unwrap();
429+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
430+
431+
assert_json_snapshot!(provider_metadata, {
432+
".runner.version" => insta::dynamic_redaction(|value,_path| {
433+
assert_eq!(value.as_str().unwrap(), VERSION.to_string());
434+
"[version]"
435+
}),
436+
});
437+
assert_json_snapshot!(run_part);
438+
},
439+
);
440+
}
441+
442+
#[test]
443+
fn test_get_run_part_no_matrix() {
444+
with_vars([("GITHUB_ACTIONS", Some("true"))], || {
445+
let github_actions_provider = GitHubActionsProvider {
446+
owner: "owner".into(),
447+
repository: "repository".into(),
448+
ref_: "refs/head/my-branch".into(),
449+
head_ref: Some("my-branch".into()),
450+
base_ref: None,
451+
sender: None,
452+
gh_data: GhData {
453+
job: "my_job".into(),
454+
run_id: "123789".into(),
455+
},
456+
event: RunEvent::Push,
457+
repository_root_path: "/home/work/my-repo".into(),
458+
};
459+
460+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
461+
462+
assert_eq!(run_part.run_id, "123789");
463+
assert_eq!(run_part.job_name, "my_job");
464+
assert_eq!(run_part.run_part_id, "my_job");
465+
assert_json_snapshot!(run_part.metadata, @"{}");
466+
})
467+
}
468+
469+
#[test]
470+
fn test_get_run_part_null_matrix() {
471+
with_vars(
472+
[
473+
("GH_MATRIX", Some("null")),
474+
(
475+
"GH_STRATEGY",
476+
Some(
477+
r#"{
478+
"fail-fast":true,
479+
"job-index":0,
480+
"job-total":1,
481+
"max-parallel":1
482+
}"#,
483+
),
484+
),
485+
],
486+
|| {
487+
let github_actions_provider = GitHubActionsProvider {
488+
owner: "owner".into(),
489+
repository: "repository".into(),
490+
ref_: "refs/head/my-branch".into(),
491+
head_ref: Some("my-branch".into()),
492+
base_ref: None,
493+
sender: None,
494+
gh_data: GhData {
495+
job: "my_job".into(),
496+
run_id: "123789".into(),
497+
},
498+
event: RunEvent::Push,
499+
repository_root_path: "/home/work/my-repo".into(),
500+
};
501+
502+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
503+
504+
assert_eq!(run_part.run_id, "123789");
505+
assert_eq!(run_part.job_name, "my_job");
506+
assert_eq!(run_part.run_part_id, "my_job");
507+
assert_json_snapshot!(run_part.metadata, @"{}");
508+
},
509+
)
510+
}
511+
512+
#[test]
513+
fn test_get_matrix_run_part() {
514+
with_vars(
515+
[
516+
(
517+
"GH_MATRIX",
518+
Some(
519+
r#"{
520+
"runner-version":"3.2.1",
521+
"numeric-value":123456789
522+
}"#,
523+
),
524+
),
525+
(
526+
"GH_STRATEGY",
527+
Some(
528+
r#"{
529+
"fail-fast":true,
530+
"job-index":1,
531+
"job-total":2,
532+
"max-parallel":2
533+
}"#,
534+
),
535+
),
536+
],
537+
|| {
538+
let github_actions_provider = GitHubActionsProvider {
539+
owner: "owner".into(),
540+
repository: "repository".into(),
541+
ref_: "refs/head/my-branch".into(),
542+
head_ref: Some("my-branch".into()),
543+
base_ref: None,
544+
sender: None,
545+
gh_data: GhData {
546+
job: "my_job".into(),
547+
run_id: "123789".into(),
548+
},
549+
event: RunEvent::Push,
550+
repository_root_path: "/home/work/my-repo".into(),
551+
};
552+
553+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
554+
555+
assert_eq!(run_part.run_id, "123789");
556+
assert_eq!(run_part.job_name, "my_job");
557+
assert_eq!(run_part.run_part_id, "my_job-{\"runner-version\":\"3.2.1\",\"numeric-value\":123456789}-{\"fail-fast\":true,\"job-index\":1,\"job-total\":2,\"max-parallel\":2}");
558+
assert_json_snapshot!(run_part.metadata, @r#"
559+
{
560+
"fail-fast": true,
561+
"job-index": 1,
562+
"job-total": 2,
563+
"max-parallel": 2,
564+
"numeric-value": 123456789,
565+
"runner-version": "3.2.1"
566+
}
567+
"#);
568+
},
569+
)
570+
}
571+
572+
#[test]
573+
fn test_get_inline_matrix_run_part() {
574+
with_vars(
575+
[
576+
(
577+
"GH_MATRIX",
578+
Some("{\"runner-version\":\"3.2.1\",\"numeric-value\":123456789}"),
579+
),
580+
(
581+
"GH_STRATEGY",
582+
Some("{\"fail-fast\":true,\"job-index\":1,\"job-total\":2,\"max-parallel\":2}"),
583+
),
584+
],
585+
|| {
586+
let github_actions_provider = GitHubActionsProvider {
587+
owner: "owner".into(),
588+
repository: "repository".into(),
589+
ref_: "refs/head/my-branch".into(),
590+
head_ref: Some("my-branch".into()),
591+
base_ref: None,
592+
sender: None,
593+
gh_data: GhData {
594+
job: "my_job".into(),
595+
run_id: "123789".into(),
596+
},
597+
event: RunEvent::Push,
598+
repository_root_path: "/home/work/my-repo".into(),
599+
};
600+
601+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
602+
603+
assert_eq!(run_part.run_id, "123789");
604+
assert_eq!(run_part.job_name, "my_job");
605+
assert_eq!(run_part.run_part_id, "my_job-{\"runner-version\":\"3.2.1\",\"numeric-value\":123456789}-{\"fail-fast\":true,\"job-index\":1,\"job-total\":2,\"max-parallel\":2}");
606+
assert_json_snapshot!(run_part.metadata, @r#"
607+
{
608+
"fail-fast": true,
609+
"job-index": 1,
610+
"job-total": 2,
611+
"max-parallel": 2,
612+
"numeric-value": 123456789,
613+
"runner-version": "3.2.1"
614+
}
615+
"#);
616+
},
617+
)
618+
}
310619
}

0 commit comments

Comments
 (0)