From 9467d8397167b39d28a67d839de89bc810b08c56 Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Mon, 17 Jun 2024 00:38:53 -0400 Subject: [PATCH] Automate file uploading to a common knowledge documents repo - Adds a drag-and-drop file upload (PDF, Markdown) - Creates a signed PR with uploaded documents in the dedicated knowledge docs repo and captures the commit SHA that is maintained with a merge commit. The SHA and file names are then populated back into the knowledge submission form. Signed-off-by: Brent Salisbury --- .env.example | 1 + src/app/api/pr/knowledge/route.ts | 4 +- src/app/api/upload/route.ts | 233 ++++++++++++++++++ .../Contribute/Knowledge/UploadFile.tsx | 166 +++++++++++++ src/components/Contribute/Knowledge/index.tsx | 168 ++++++++++--- .../Contribute/Knowledge/knowledge.css | 8 +- 6 files changed, 544 insertions(+), 36 deletions(-) create mode 100644 src/app/api/upload/route.ts create mode 100644 src/components/Contribute/Knowledge/UploadFile.tsx diff --git a/.env.example b/.env.example index 572ae234..02fa3123 100644 --- a/.env.example +++ b/.env.example @@ -11,3 +11,4 @@ IL_MERLINITE_MODEL_NAME= GITHUB_TOKEN= TAXONOMY_REPO_OWNER= TAXONOMY_REPO= +TAXONOMY_DOCUMENTS_REPO=github.com// diff --git a/src/app/api/pr/knowledge/route.ts b/src/app/api/pr/knowledge/route.ts index df9cb0fa..8cd0e386 100644 --- a/src/app/api/pr/knowledge/route.ts +++ b/src/app/api/pr/knowledge/route.ts @@ -103,7 +103,7 @@ Creator names: ${creators} // Create a new branch in the user's fork await createBranch(headers, githubUsername, branchName, baseBranchSha); - // Create both files in a single commit + // Create both files in a single commit with DCO sign-off await createFilesInSingleCommit( headers, githubUsername, @@ -112,7 +112,7 @@ Creator names: ${creators} { path: newAttributionFilePath, content: attributionContent } ], branchName, - task_details + `${task_details}\n\nSigned-off-by: ${email}` ); // Create a pull request from the user's fork to the upstream repository diff --git a/src/app/api/upload/route.ts b/src/app/api/upload/route.ts new file mode 100644 index 00000000..f86e96d3 --- /dev/null +++ b/src/app/api/upload/route.ts @@ -0,0 +1,233 @@ +// src/app/api/upload/route.ts +import { NextResponse } from 'next/server'; +import { getToken } from 'next-auth/jwt'; +import { NextRequest } from 'next/server'; + +const GITHUB_API_URL = 'https://api.github.com'; +const TAXONOMY_DOCUMENTS_REPO = process.env.TAXONOMY_DOCUMENTS_REPO!; +const BASE_BRANCH = 'main'; + +export async function POST(req: NextRequest) { + const token = await getToken({ req, secret: process.env.NEXTAUTH_SECRET! }); + console.log('GitHub Token:', token); + + if (!token || !token.accessToken) { + console.error('Unauthorized: Missing or invalid access token'); + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + } + + const githubToken = token.accessToken as string; + const headers = { + 'Content-Type': 'application/json', + Authorization: `Bearer ${githubToken}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28' + }; + + try { + const body = await req.json(); + const { files } = body; + + // Fetch GitHub username and email + const { githubUsername, userEmail } = await getGitHubUsernameAndEmail(headers); + console.log('GitHub Username:', githubUsername); + console.log('User Email:', userEmail); + + const repoOwner = githubUsername; + const repoName = TAXONOMY_DOCUMENTS_REPO.split('/').pop(); + + if (!repoName) { + throw new Error('Repository name is undefined'); + } + + const newBranchName = `upload-${Date.now()}`; + + // Get the base branch SHA + const baseBranchSha = await getBranchSha(headers, repoOwner, repoName, BASE_BRANCH); + console.log(`Base branch SHA: ${baseBranchSha}`); + + // Create a new branch + await createBranch(headers, repoOwner, repoName, newBranchName, baseBranchSha); + + // Create files in the new branch + const commitSha = await createFilesCommit(headers, repoOwner, repoName, newBranchName, files, userEmail); + + // Create a pull request + const prUrl = await createPullRequest( + headers, + repoOwner, + repoName, + newBranchName, + files.map((file: { fileName: string }) => file.fileName).join(', ') + ); + + return NextResponse.json( + { + repoUrl: `https://github.com/${repoOwner}/${repoName}`, + commitSha, + documentNames: files.map((file: { fileName: string }) => file.fileName), + prUrl + }, + { status: 201 } + ); + } catch (error) { + console.error('Failed to upload documents:', error); + return NextResponse.json({ error: 'Failed to upload documents' }, { status: 500 }); + } +} + +async function getGitHubUsernameAndEmail(headers: HeadersInit): Promise<{ githubUsername: string; userEmail: string }> { + const response = await fetch(`${GITHUB_API_URL}/user`, { headers }); + + if (!response.ok) { + const errorText = await response.text(); + console.error('Failed to fetch GitHub username and email:', response.status, errorText); + throw new Error('Failed to fetch GitHub username and email'); + } + + const data = await response.json(); + return { githubUsername: data.login, userEmail: data.email }; +} + +async function getBranchSha(headers: HeadersInit, owner: string, repo: string, branch: string): Promise { + const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/ref/heads/${branch}`, { headers }); + + if (!response.ok) { + const errorText = await response.text(); + console.error('Failed to get branch SHA:', response.status, errorText); + throw new Error('Failed to get branch SHA'); + } + + const data = await response.json(); + return data.object.sha; +} + +async function createBranch(headers: HeadersInit, owner: string, repo: string, branchName: string, baseSha: string) { + const body = JSON.stringify({ + ref: `refs/heads/${branchName}`, + sha: baseSha + }); + + const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/refs`, { + method: 'POST', + headers, + body + }); + + if (!response.ok) { + const errorText = await response.text(); + console.error('Failed to create branch:', response.status, errorText); + throw new Error('Failed to create branch'); + } +} + +async function createFilesCommit( + headers: HeadersInit, + owner: string, + repo: string, + branchName: string, + files: { fileName: string; fileContent: string }[], + userEmail: string +): Promise { + // Create blobs for each file + const blobs = await Promise.all( + files.map((file) => + fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/blobs`, { + method: 'POST', + headers, + body: JSON.stringify({ + content: file.fileContent, + encoding: 'utf-8' + }) + }).then((response) => response.json()) + ) + ); + + // Get base tree + const baseTreeSha = await getBaseTreeSha(headers, owner, repo, branchName); + + // Create tree + const createTreeResponse = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/trees`, { + method: 'POST', + headers, + body: JSON.stringify({ + base_tree: baseTreeSha, + tree: files.map((file, index) => ({ + path: file.fileName, + mode: '100644', + type: 'blob', + sha: blobs[index].sha + })) + }) + }); + + if (!createTreeResponse.ok) { + const errorText = await createTreeResponse.text(); + console.error('Failed to create tree:', createTreeResponse.status, errorText); + throw new Error('Failed to create tree'); + } + + const treeData = await createTreeResponse.json(); + + // Create commit with DCO sign-off + const createCommitResponse = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/commits`, { + method: 'POST', + headers, + body: JSON.stringify({ + message: `Add files: ${files.map((file) => file.fileName).join(', ')}\n\nSigned-off-by: ${userEmail}`, + tree: treeData.sha, + parents: [await getBranchSha(headers, owner, repo, branchName)] + }) + }); + + if (!createCommitResponse.ok) { + const errorText = await createCommitResponse.text(); + console.error('Failed to create commit:', createCommitResponse.status, errorText); + throw new Error('Failed to create commit'); + } + + const commitData = await createCommitResponse.json(); + + // Update branch reference + await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/refs/heads/${branchName}`, { + method: 'PATCH', + headers, + body: JSON.stringify({ sha: commitData.sha }) + }); + + return commitData.sha; +} + +async function getBaseTreeSha(headers: HeadersInit, owner: string, repo: string, branch: string): Promise { + const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/trees/${branch}`, { headers }); + + if (!response.ok) { + const errorText = await response.text(); + console.error('Failed to get base tree SHA:', response.status, errorText); + throw new Error('Failed to get base tree SHA'); + } + + const data = await response.json(); + return data.sha; +} + +async function createPullRequest(headers: HeadersInit, owner: string, repo: string, branchName: string, fileNames: string): Promise { + const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/pulls`, { + method: 'POST', + headers, + body: JSON.stringify({ + title: `Add files: ${fileNames}`, + head: branchName, + base: BASE_BRANCH + }) + }); + + if (!response.ok) { + const errorText = await response.text(); + console.error('Failed to create pull request:', response.status, errorText); + throw new Error('Failed to create pull request'); + } + + const data = await response.json(); + return data.html_url; +} diff --git a/src/components/Contribute/Knowledge/UploadFile.tsx b/src/components/Contribute/Knowledge/UploadFile.tsx new file mode 100644 index 00000000..75eae440 --- /dev/null +++ b/src/components/Contribute/Knowledge/UploadFile.tsx @@ -0,0 +1,166 @@ +// src/components/Contribute/Knowledge/UploadFile.tsx +'use client'; +import React, { useState, useEffect } from 'react'; +import { + MultipleFileUploadStatusItem, + MultipleFileUploadStatus, + MultipleFileUpload, + MultipleFileUploadMain +} from '@patternfly/react-core/dist/dynamic/components/MultipleFileUpload'; +import { Modal } from '@patternfly/react-core/dist/dynamic/next/components/Modal'; +import UploadIcon from '@patternfly/react-icons/dist/esm/icons/upload-icon'; +import { ExclamationTriangleIcon } from '@patternfly/react-icons/dist/dynamic/icons/exclamation-triangle-icon'; +import { FileRejection, DropEvent } from 'react-dropzone'; +import { Button } from '@patternfly/react-core/dist/dynamic/components/Button'; +import { HelperText, HelperTextItem } from '@patternfly/react-core/dist/dynamic/components/HelperText'; + +interface readFile { + fileName: string; + data?: string; + loadResult?: 'danger' | 'success'; + loadError?: DOMException; +} + +export const UploadFile: React.FunctionComponent<{ onFilesChange: (files: File[]) => void }> = ({ onFilesChange }) => { + const [currentFiles, setCurrentFiles] = useState([]); + const [readFileData, setReadFileData] = useState([]); + const [showStatus, setShowStatus] = useState(false); + const [statusIcon, setStatusIcon] = useState<'inProgress' | 'success' | 'danger'>('inProgress'); + const [modalText, setModalText] = useState(''); + + useEffect(() => { + if (currentFiles.length > 0) { + setShowStatus(true); + } else { + setShowStatus(false); + } + }, [currentFiles]); + + useEffect(() => { + if (readFileData.length < currentFiles.length) { + setStatusIcon('inProgress'); + } else if (readFileData.every((file) => file.loadResult === 'success')) { + setStatusIcon('success'); + } else { + setStatusIcon('danger'); + } + }, [readFileData, currentFiles]); + + const removeFiles = (namesOfFilesToRemove: string[]) => { + const newCurrentFiles = currentFiles.filter((file) => !namesOfFilesToRemove.includes(file.name)); + const newReadFiles = readFileData.filter((file) => !namesOfFilesToRemove.includes(file.fileName)); + setCurrentFiles(newCurrentFiles); + setReadFileData(newReadFiles); + }; + + const handleFileDrop = (_event: DropEvent, droppedFiles: File[]) => { + const currentFileNames = currentFiles.map((file) => file.name); + const reUploads = droppedFiles.filter((file) => currentFileNames.includes(file.name)); + + const newFiles = [ + ...currentFiles.filter((file) => !reUploads.includes(file)), + ...droppedFiles.filter((file) => !currentFileNames.includes(file.name)) + ]; + setCurrentFiles(newFiles); + onFilesChange(newFiles); + }; + + const handleReadSuccess = (data: string, file: File) => { + setReadFileData((prevReadFiles) => { + const existingFile = prevReadFiles.find((readFile) => readFile.fileName === file.name); + if (existingFile) { + return prevReadFiles; + } + return [...prevReadFiles, { data, fileName: file.name, loadResult: 'success' }]; + }); + }; + + const handleReadFail = (error: DOMException, file: File) => { + setReadFileData((prevReadFiles) => { + const existingFile = prevReadFiles.find((readFile) => readFile.fileName === file.name); + if (existingFile) { + return prevReadFiles; + } + return [...prevReadFiles, { loadError: error, fileName: file.name, loadResult: 'danger' }]; + }); + }; + + const handleDropRejected = (fileRejections: FileRejection[]) => { + console.warn('Files rejected:', fileRejections); + if (fileRejections.length === 1) { + setModalText(`${fileRejections[0].file.name} is not an accepted file type`); + } else { + const rejectedMessages = fileRejections.reduce((acc, fileRejection) => (acc += `${fileRejection.file.name}, `), ''); + setModalText(`${rejectedMessages} are not accepted file types`); + } + }; + + const createHelperText = (file: File) => { + const fileResult = readFileData.find((readFile) => readFile.fileName === file.name); + if (fileResult?.loadError) { + return ( + + {fileResult.loadError.toString()} + + ); + } + }; + + const successfullyReadFileCount = readFileData.filter((fileData) => fileData.loadResult === 'success').length; + console.log('Successfully read file count:', successfullyReadFileCount); + console.log('Current files count:', currentFiles.length); + + return ( + <> + + } + titleText="Drag and drop files here" + titleTextSeparator="or" + infoText="Accepted file types: PDF, Markdown" + /> + {showStatus && ( + + {currentFiles.map((file) => ( + removeFiles([file.name])} + onReadSuccess={handleReadSuccess} + onReadFail={handleReadFail} + progressHelperText={createHelperText(file)} + /> + ))} + + )} + setModalText('')} + > +
+ {modalText} +
+ + + + + ); +}; diff --git a/src/components/Contribute/Knowledge/index.tsx b/src/components/Contribute/Knowledge/index.tsx index bd8253c5..f628544a 100644 --- a/src/components/Contribute/Knowledge/index.tsx +++ b/src/components/Contribute/Knowledge/index.tsx @@ -13,8 +13,10 @@ import { TextArea } from '@patternfly/react-core/dist/dynamic/components/TextAre import { PlusIcon, MinusCircleIcon } from '@patternfly/react-icons/dist/dynamic/icons/'; import yaml from 'js-yaml'; import { validateFields, validateEmail, validateUniqueItems } from '../../../utils/validation'; +import { UploadFile } from './UploadFile'; export const KnowledgeForm: React.FunctionComponent = () => { + // Define the initial state and type const [email, setEmail] = useState(''); const [name, setName] = useState(''); const [task_description, setTaskDescription] = useState(''); @@ -37,10 +39,14 @@ export const KnowledgeForm: React.FunctionComponent = () => { const [isFailureAlertVisible, setIsFailureAlertVisible] = useState(false); const [failure_alert_title, setFailureAlertTitle] = useState(''); - const [failure_alert_message, setFailureAlertMessage] = useState(''); + const [failure_alert_message, setFailureAlertMessage] = useState(''); const [success_alert_title, setSuccessAlertTitle] = useState(''); - const [success_alert_message, setSuccessAlertMessage] = useState(''); + const [success_alert_message, setSuccessAlertMessage] = useState(''); + const [successAlertLink, setSuccessAlertLink] = useState(''); + + const [useFileUpload, setUseFileUpload] = useState(false); + const [uploadedFiles, setUploadedFiles] = useState([]); const handleInputChange = (index: number, type: string, value: string) => { switch (type) { @@ -89,6 +95,7 @@ export const KnowledgeForm: React.FunctionComponent = () => { setLicenseWork(''); setCreators(''); setRevision(''); + setUploadedFiles([]); }; const onCloseSuccessAlert = () => { @@ -99,6 +106,11 @@ export const KnowledgeForm: React.FunctionComponent = () => { setIsFailureAlertVisible(false); }; + const handleFilesChange = (files: File[]) => { + setUploadedFiles(files); + setPatterns(files.map((file) => file.name).join(', ')); // Populate the patterns field + }; + const handleSubmit = async (event: React.FormEvent) => { event.preventDefault(); @@ -178,18 +190,74 @@ export const KnowledgeForm: React.FunctionComponent = () => { const result = await response.json(); setSuccessAlertTitle('Knowledge contribution submitted successfully!'); - setSuccessAlertMessage(result.html_url); + setSuccessAlertMessage('A pull request containing your knowledge submission has been successfully created.'); + setSuccessAlertLink(result.html_url); setIsSuccessAlertVisible(true); resetForm(); } catch (error: unknown) { if (error instanceof Error) { - setFailureAlertTitle('Failed to submit your Knowledge contribution!'); + setFailureAlertTitle('Failed to submit your Knowledge contribution'); setFailureAlertMessage(error.message); setIsFailureAlertVisible(true); } } }; + const handleDocumentUpload = async () => { + if (uploadedFiles.length > 0) { + const fileContents: { fileName: string; fileContent: string }[] = []; + + await Promise.all( + uploadedFiles.map( + (file) => + new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = (e) => { + const fileContent = e.target!.result as string; + fileContents.push({ fileName: file.name, fileContent }); + resolve(); + }; + reader.onerror = reject; + reader.readAsText(file); + }) + ) + ); + + if (fileContents.length === uploadedFiles.length) { + try { + const response = await fetch('/api/upload', { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ files: fileContents }) + }); + + const result = await response.json(); + if (response.ok) { + setRepo(result.repoUrl); + setCommit(result.commitSha); + setPatterns(result.documentNames.join(', ')); // Populate the patterns field + console.log('Files uploaded:', result.documentNames); + setSuccessAlertTitle('Document uploaded successfully!'); + setSuccessAlertMessage('The document has been uploaded and a PR has been created.'); + setSuccessAlertLink(result.prUrl); + setIsSuccessAlertVisible(true); + setUseFileUpload(false); // Switch back to manual mode + } else { + throw new Error(result.error || 'Failed to upload document'); + } + } catch (error: unknown) { + if (error instanceof Error) { + setFailureAlertTitle('Failed to upload document'); + setFailureAlertMessage(error.message); + setIsFailureAlertVisible(true); + } + } + } + } + }; + const handleDownloadYaml = () => { const infoFields = { email, name, task_description, task_details, domain, repo, commit, patterns }; const attributionFields = { title_work, link_work, revision, license_work, creators }; @@ -408,33 +476,62 @@ Creator names: ${creators} } > - - setRepo(value)} - /> - setCommit(value)} - /> - setPatterns(value)} - /> + +
+ + +
+ + {!useFileUpload ? ( + + setRepo(value)} + /> + setCommit(value)} + /> + setPatterns(value)} + /> + + ) : ( + <> + + + + )} + } actionLinks={ - - View your pull request - + <> + + View it here + + } > - Thank you for your contribution! + {success_alert_message} )} {isFailureAlertVisible && ( @@ -506,6 +605,7 @@ Creator names: ${creators} {failure_alert_message} )} +