Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fetchpost: add formdata to report #434

Merged
merged 3 commits into from
Aug 14, 2022
Merged

fetchpost: add formdata to report #434

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5e63019
`fetchpost`: add formdata to fetchpost report
jqnatividad Aug 14, 2022
1d766c0
add `fetchpost` report test
jqnatividad Aug 14, 2022
01e5f63
add test fetchpost-report.tsv
jqnatividad Aug 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions resources/test/data.csv.fetchpost-report.tsv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
url form status cache_hit retries elapsed_ms response
https://httpbin.org/post "{""bool_col"": String(""true""), ""col1"": String(""a""), ""number_col"": String(""42"")}" 200 0 0 171 "{""bool_col"": String(""true""), ""col1"": String(""a""), ""number_col"": String(""42"")}"
https://httpbin.org/post "{""bool_col"": String(""false""), ""col1"": String(""b""), ""number_col"": String(""3.14"")}" 200 0 0 33 "{""bool_col"": String(""false""), ""col1"": String(""b""), ""number_col"": String(""3.14"")}"
https://httpbin.org/post "{""bool_col"": String(""true""), ""col1"": String(""c""), ""number_col"": String(""666"")}" 200 0 0 20 "{""bool_col"": String(""true""), ""col1"": String(""c""), ""number_col"": String(""666"")}"
22 changes: 11 additions & 11 deletions src/cmd/fetchpost.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,12 +135,13 @@ Fetch options:
--report <d|s> Creates a report of the fetchpost job. The report has the same name as the input file
with the ".fetchpost-report" suffix.
There are two kinds of report - d for "detailed" & s for "short". The detailed report
has the same columns as the input CSV with six additional columns -
qsv_fetchp_url, qsv_fetchp_status, qsv_fetchp_cache_hit, qsv_fetchp_retries,
qsv_fetchp_elapsed_ms & qsv_fetchp_response.
fetchp_url - URL used, fetchp_status - HTTP status code, fetchp_cache_hit - cached hit flag,
fetchp_retries - retry attempts, fetchp_elapsed - elapsed time & fetchp_response - the response.
The short report only has the six columns without the "qsv_fetchp_" column name prefix.
has the same columns as the input CSV with seven additional columns -
qsv_fetchp_url, qsv_fetchp_form, qsv_fetchp_status, qsv_fetchp_cache_hit,
qsv_fetchp_retries, qsv_fetchp_elapsed_ms & qsv_fetchp_response.
fetchp_url - URL used, qsv_fetchp_form - form data sent, fetchp_status - HTTP status code,
fetchp_cache_hit - cached hit flag, fetchp_retries - retry attempts,
fetchp_elapsed - elapsed time & fetchp_response - the response.
The short report only has the sevenn columns without the "qsv_fetchp_" column name prefix.
--redis Use Redis to cache responses. It connects to "redis://127.0.0.1:6379/2"
with a connection pool size of 20, with a TTL of 28 days, and a cache hit
NOT renewing an entry's TTL.
Expand Down Expand Up @@ -447,6 +448,7 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
let mut report_wtr;
let report_path;
if report == ReportKind::None {
// no report, point report_wtr to /dev/null (AKA sink)
report_wtr = Config::new(&Some("sink".to_string())).writer()?;
report_path = "".to_string();
} else {
Expand All @@ -467,6 +469,7 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
""
};
report_headers.push_field(format!("{rptcol_prefix}url").as_bytes());
report_headers.push_field(format!("{rptcol_prefix}form").as_bytes());
report_headers.push_field(format!("{rptcol_prefix}status").as_bytes());
report_headers.push_field(format!("{rptcol_prefix}cache_hit").as_bytes());
report_headers.push_field(format!("{rptcol_prefix}retries").as_bytes());
Expand Down Expand Up @@ -638,6 +641,7 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
report_record.clear();
}
report_record.push_field(url.as_bytes());
report_record.push_field(format!("{form_body_jsonmap:?}").as_bytes());
report_record.push_field(final_response.status_code.to_string().as_bytes());
report_record.push_field(if was_cached { b"1" } else { b"0" });
report_record.push_field(final_response.retries.to_string().as_bytes());
Expand All @@ -655,11 +659,7 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
}
}

if report == ReportKind::None {
drop(report_wtr);
} else {
report_wtr.flush()?;
}
report_wtr.flush()?;

if not_quiet {
if args.flag_redis {
Expand Down
41 changes: 41 additions & 0 deletions tests/test_fetch.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -690,3 +690,44 @@ fn fetchpost_literalurl_test() {

assert_eq!(got_parsed, expected);
}

#[test]
fn fetchpost_simple_report() {
let wrk = Workdir::new("fetchpost_simple_report");
wrk.create(
"data.csv",
vec![
svec!["col1", "number_col", "bool_col"],
svec!["a", "42", "true"],
svec!["b", "3.14", "false"],
svec!["c", "666", "true"],
],
);
let mut cmd = wrk.command("fetchpost");
cmd.arg("https://httpbin.org/post")
.arg("bool_col,col1,number_col")
.arg("--jql")
.arg(r#""form""#)
.arg("--new-column")
.arg("response")
.arg("--report")
.arg("short")
.arg("data.csv");

let mut cmd = wrk.command("index");
cmd.arg("data.csv.fetchpost-report.tsv");

let mut cmd = wrk.command("select");
cmd.arg("url,form,status,cache_hit,retries,response")
.arg(wrk.load_test_file("data.csv.fetchpost-report.tsv"));

let got: Vec<Vec<String>> = wrk.read_stdout(&mut cmd);
let expected = vec![
svec!["url", "form", "status", "cache_hit", "retries", "response"],
svec!["https://httpbin.org/post", "{\"bool_col\": String(\"true\"), \"col1\": String(\"a\"), \"number_col\": String(\"42\")}", "200", "0", "0", "{\"bool_col\": String(\"true\"), \"col1\": String(\"a\"), \"number_col\": String(\"42\")}"],
svec!["https://httpbin.org/post", "{\"bool_col\": String(\"false\"), \"col1\": String(\"b\"), \"number_col\": String(\"3.14\")}", "200", "0", "0", "{\"bool_col\": String(\"false\"), \"col1\": String(\"b\"), \"number_col\": String(\"3.14\")}"],
svec!["https://httpbin.org/post", "{\"bool_col\": String(\"true\"), \"col1\": String(\"c\"), \"number_col\": String(\"666\")}", "200", "0", "0", "{\"bool_col\": String(\"true\"), \"col1\": String(\"c\"), \"number_col\": String(\"666\")}"],
];

assert_eq!(got, expected);
}