From 6ec4200b7b954cd47fa768e4a142945313c42106 Mon Sep 17 00:00:00 2001 From: Connor Campbell Date: Mon, 7 Oct 2024 19:39:11 +0900 Subject: [PATCH 1/3] refactor: delete only open issues on reimport + use upserts --- src/import/src/main.rs | 19 +++++++++---- src/shared/src/functions.rs | 56 +++++++++++++++++++++++++++++++++++-- src/shared/src/types.rs | 13 +++++++++ 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/src/import/src/main.rs b/src/import/src/main.rs index 5006894..cd907ae 100644 --- a/src/import/src/main.rs +++ b/src/import/src/main.rs @@ -12,11 +12,20 @@ async fn function_handler(event: Request) -> Result, Error> { let mut tx: sqlx::Transaction<'_, sqlx::Postgres> = pool.begin().await?; - // allow the import to be completely rerun by removing all data - sqlx::query("DELETE FROM projects WHERE slug = $1") - .bind(&project.slug) - .execute(&mut *tx) - .await?; + // delete all open issues related to a preexisting project + sqlx::query( + r#" + DELETE FROM issues + WHERE repository_id IN ( + SELECT repositories.id + FROM repositories + JOIN projects ON repositories.project_id = projects.id + WHERE projects.slug = $1) + AND issues.open = true"#, + ) + .bind(&project.slug) + .execute(&mut *tx) + .await?; let project_id = insert_project(&project, &mut tx).await?; diff --git a/src/shared/src/functions.rs b/src/shared/src/functions.rs index 8ed0351..ee570db 100644 --- a/src/shared/src/functions.rs +++ b/src/shared/src/functions.rs @@ -103,10 +103,56 @@ pub async fn import_repositories( let repo_id: i32 = repo_row.get("id"); + // let mut pages_remaining = true; + // let mut all_repo_issues: Vec = vec![]; + // let mut page_number: u32 = 1; + + // info!("\n\n CURRENT REPO BEING IMPORTED: {}\n\n", repo_info.name); + + // while pages_remaining { + // let page = octocrab + // .issues(&repo_info.owner, &repo_info.name) + // .list() + // .state(State::Open) + // .per_page(100) + // .page(page_number) + // .send() + // .await?; + + // info!("\n\n NEXT PAGE ISSUES COUNT: {}\n\n", page.items.len()); + + // all_repo_issues.extend(page.items); + // pages_remaining = page.next.is_some(); + // page_number += 1; + // tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; + // } + + // info!( + // "\n\nREPO NAME: {}, ALL REPO ISSUES/PULLS COUNT: {}\n\n", + // &repo_info.name, + // &all_repo_issues.len() + // ); + + // let filtered_issues: Vec = all_repo_issues + // .into_iter() + // .filter_map(|issue| { + // issue + // .pull_request + // .is_none() + // .then(|| KudosIssue::from(issue)) + // }) + // .collect(); + + // info!( + // "\n\nREPO NAME: {}, ALL FILTERED ISSUES COUNT: {}\n\n", + // &repo_info.name, + // &filtered_issues.len() + // ); + let stream = octocrab - .issues(repo_info.owner, repo_info.name) + .issues(&repo_info.owner, &repo_info.name) .list() - .state(State::All) + .state(State::Open) .per_page(100) .send() .await? @@ -123,6 +169,12 @@ pub async fn import_repositories( .try_collect() .await?; + info!( + "\n\nREPO NAME: {}, ALL FILTERED ISSUES COUNT: {}\n\n", + &repo_info.name, + &filtered_issues.len() + ); + if filtered_issues.is_empty() { continue; } diff --git a/src/shared/src/types.rs b/src/shared/src/types.rs index de3aa39..6ab443d 100644 --- a/src/shared/src/types.rs +++ b/src/shared/src/types.rs @@ -70,6 +70,12 @@ impl Repository { let query_string = r#" INSERT INTO repositories (slug, name, url, language_slug, project_id) VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (url) + DO UPDATE SET + slug = EXCLUDED.slug, + name = EXCLUDED.name, + project_id = EXCLUDED.project_id, + language_slug = EXCLUDED.language_slug RETURNING id; "#; query_string @@ -106,6 +112,13 @@ impl Project { let query_string = r#" INSERT INTO projects (name, slug, types, purposes, stack_levels, technologies) VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (slug) + DO UPDATE SET + name = EXCLUDED.name, + types = EXCLUDED.types, + purposes = EXCLUDED.purposes, + stack_levels = EXCLUDED.stack_levels, + technologies = EXCLUDED.technologies RETURNING id; "#; query_string From 95f8c3030ff1d4cefdc3c884e5eed66c74eac59c Mon Sep 17 00:00:00 2001 From: CJ13th <48095175+CJ13th@users.noreply.github.com> Date: Tue, 8 Oct 2024 13:12:10 +0100 Subject: [PATCH 2/3] Update add_project.yaml --- .github/ISSUE_TEMPLATE/add_project.yaml | 90 ++++++++++++++----------- 1 file changed, 50 insertions(+), 40 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/add_project.yaml b/.github/ISSUE_TEMPLATE/add_project.yaml index 9f66563..9dd1f36 100644 --- a/.github/ISSUE_TEMPLATE/add_project.yaml +++ b/.github/ISSUE_TEMPLATE/add_project.yaml @@ -33,57 +33,49 @@ body: validations: required: true - - type: input + - type: textarea id: websites attributes: label: Websites - description: "Enter website URLs separated by commas." + description: "On each new line write the website URLs" placeholder: "https://morekudos.com" - validations: - required: true - - type: input + - type: textarea id: documentation attributes: label: Documentation - description: "Enter documentation URLs separated by commas." - placeholder: "https://github.com/kudos-ink" - validations: - required: true + description: "On each new line write the documentation URLs" + placeholder: "https://github.com/kudos-ink/docs" - - type: input + - type: textarea id: explorers attributes: label: Explorers - description: "Enter explorer URLs separated by commas." - validations: - required: true + description: "On each new line write the explorer URLs." + placeholder: "https://polkadot.subscan.io/" - - type: input + - type: textarea id: repositories attributes: label: Repositories - description: "Enter repository URLs separated by commas." - placeholder: "https://github.com/kudos-ink/portal, https://github.com/kudos-ink/issues-api" + description: "On each new line write the repository URLs with front end friendly label." + placeholder: "LABEL: Kudos Portal, URL: https://github.com/kudos-ink/portal" validations: required: true - - type: input - id: socialMedia + - type: textarea + id: social attributes: label: Social Media - description: "Enter social media URLs separated by commas." + description: "On each new line write the social media URLs." placeholder: "https://x.com/KudosPortal" - validations: - required: true - - type: input + - type: textarea id: networks attributes: label: Networks - description: "Enter network names separated by commas." - validations: - required: true + description: "On each new line write the relevant networks." + placeholder: "Polkadot\nEthereum\nAstar" - type: checkboxes id: purposes @@ -91,28 +83,40 @@ body: label: Purposes description: "Select the purposes." options: - - label: Data - validations: - required: true + - label: ai + - label: bridge + - label: dao + - label: data + - label: defi + - label: evm + - label: gaming + - label: governance + - label: nft + - label: oracles + - label: rwa + - label: socialfi + - label: staking + - label: testing + - label: universal - - type: input + - type: checkboxes id: stackLevels attributes: label: Stack Levels description: "Enter stack levels separated by commas." - validations: - required: true + options: + - label: protocol + - label: runtime + - label: smart-contract + - label: messaging + - label: offchain - - type: checkboxes + - type: textarea id: technologies attributes: label: Technologies - description: "Select the technologies used." - options: - - label: TypeScript - - label: Rust - validations: - required: true + description: "On each line write the technologies used." + placeholder: "Substrate\nInk!" - type: checkboxes id: types @@ -120,6 +124,12 @@ body: label: Types description: "Select the types." options: + - label: platform + - label: tool + - label: library + - label: client + - label: framework + - label: template + - label: mobile + - label: service - label: dApp - validations: - required: true From 75ea0779151da20d03878777f47f96dccdfac73a Mon Sep 17 00:00:00 2001 From: CJ13th <48095175+CJ13th@users.noreply.github.com> Date: Tue, 8 Oct 2024 13:21:29 +0100 Subject: [PATCH 3/3] Update add_project.yaml --- .github/ISSUE_TEMPLATE/add_project.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/add_project.yaml b/.github/ISSUE_TEMPLATE/add_project.yaml index 9dd1f36..2027e46 100644 --- a/.github/ISSUE_TEMPLATE/add_project.yaml +++ b/.github/ISSUE_TEMPLATE/add_project.yaml @@ -59,7 +59,7 @@ body: attributes: label: Repositories description: "On each new line write the repository URLs with front end friendly label." - placeholder: "LABEL: Kudos Portal, URL: https://github.com/kudos-ink/portal" + placeholder: "Kudos Portal - https://github.com/kudos-ink/portal" validations: required: true