From 4199162bbdf8245d72f9e9d809504c2055c2d39e Mon Sep 17 00:00:00 2001 From: Samuel Valdes Gutierrez Date: Tue, 16 Apr 2024 15:58:37 -0400 Subject: [PATCH] [OSCI][FEAT] Changelog Project - PoC Changelog and release notes automation tool - OpenSearch Dashboards (#5519) Refactor and Enhance Workflow Management - Added and updated changesets for multiple PRs to improve tracking and documentation of changes. - Removed unnecessary test and dummy files (`test.txt`, various `.yml` fragments) to clean up the repository. - Refactored workflow scripts to streamline changelog generation and fragment handling, moving temporary files to a designated folder. - Updated GitHub Actions workflows by changing event triggers from `pull_request` to `pull_request_target` and vice versa to optimize workflow execution. - Enhanced security and automation by updating token names and adding write permissions to the changeset workflow. - Deleted obsolete workflow file for creating changeset files, now handled by an automated process. - Major clean-up of dummy fragment files and unnecessary changelog entries to maintain clarity and relevancy in documentation. - Implemented minor updates and improvements in codebase, specifically in generating release notes and handling fragments. --------- Signed-off-by: Johnathon Bowers Signed-off-by: CMDWillYang Signed-off-by: Qiwen Li Signed-off-by: qiwen li Signed-off-by: Samuel Valdes Gutierrez Signed-off-by: Ashwin P Chandran Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Johnathon Bowers Co-authored-by: CMDWillYang Co-authored-by: Qiwen Li Co-authored-by: Ashwin P Chandran Co-authored-by: Anan Zhuang Co-authored-by: Josh Romero Co-authored-by: autochangeset[bot] <154024398+autochangeset[bot]@users.noreply.github.com> Co-authored-by: opensearch-bot[bot] <154024398+opensearch-bot[bot]@users.noreply.github.com> Co-authored-by: opensearch-bot-dev[bot] <154634848+opensearch-bot-dev[bot]@users.noreply.github.com> Co-authored-by: Ashwin P Chandran Co-authored-by: Miki --- .github/pull_request_template.md | 12 ++ .github/workflows/changelog_verifier.yml | 19 --- .../opensearch_changelog_workflow.yml | 23 +++ changelogs/README.md | 5 + package.json | 1 + scripts/generate_release_note.js | 8 ++ src/dev/generate_release_note.ts | 134 ++++++++++++++++++ src/dev/generate_release_note_helper.ts | 59 ++++++++ 8 files changed, 242 insertions(+), 19 deletions(-) delete mode 100644 .github/workflows/changelog_verifier.yml create mode 100644 .github/workflows/opensearch_changelog_workflow.yml create mode 100644 changelogs/README.md create mode 100644 scripts/generate_release_note.js create mode 100644 src/dev/generate_release_note.ts create mode 100644 src/dev/generate_release_note_helper.ts diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 3cb6f172b119..662ca15b8d68 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -19,6 +19,18 @@ the functionality of your change --> +## Changelog + + ### Check List - [ ] All tests pass diff --git a/.github/workflows/changelog_verifier.yml b/.github/workflows/changelog_verifier.yml deleted file mode 100644 index 0890ea8b8fbb..000000000000 --- a/.github/workflows/changelog_verifier.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: "Changelog Verifier" -on: - pull_request: - branches: [ '**', '!feature/**' ] - types: [opened, edited, review_requested, synchronize, reopened, ready_for_review, labeled, unlabeled] - -jobs: - # Enforces the update of a changelog file on every pull request - verify-changelog: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - ref: ${{ github.event.pull_request.head.sha }} - - - uses: dangoslen/changelog-enforcer@v3 - with: - skipLabels: "autocut, Skip-Changelog" diff --git a/.github/workflows/opensearch_changelog_workflow.yml b/.github/workflows/opensearch_changelog_workflow.yml new file mode 100644 index 000000000000..8af8b0d70b0d --- /dev/null +++ b/.github/workflows/opensearch_changelog_workflow.yml @@ -0,0 +1,23 @@ +name: OpenSearch Changelog Workflow + +on: + pull_request_target: + types: [opened, reopened, edited] + +permissions: + contents: read + issues: write + pull-requests: write + +jobs: + update-changelog: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + - name: Parse changelog entries and submit request for changset creation + uses: BigSamu/OpenSearch_Changelog_Workflow@1.0.0-alpha1 + with: + token: ${{secrets.GITHUB_TOKEN}} + CHANGELOG_PR_BRIDGE_URL_DOMAIN: ${{secrets.CHANGELOG_PR_BRIDGE_URL_DOMAIN}} + CHANGELOG_PR_BRIDGE_API_KEY: ${{secrets.CHANGELOG_PR_BRIDGE_API_KEY}} diff --git a/changelogs/README.md b/changelogs/README.md new file mode 100644 index 000000000000..a4620754cfd1 --- /dev/null +++ b/changelogs/README.md @@ -0,0 +1,5 @@ +# Changelog and Release Notes + +For information regarding the changelog and release notes process, please consult the README in the GitHub Actions repository that this process utilizes. To view this README, follow the link below: + +[GitHub Actions Workflow README](https://github.com/BigSamu/OpenSearch_Change_Set_Create_Action/blob/main/README.md) diff --git a/package.json b/package.json index 172c48cfbd70..c728618fcbcd 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "osd:bootstrap": "scripts/use_node scripts/build_ts_refs && scripts/use_node scripts/register_git_hook", "spec_to_console": "scripts/use_node scripts/spec_to_console", "pkg-version": "scripts/use_node -e \"console.log(require('./package.json').version)\"", + "release_note:generate": "scripts/use_node scripts/generate_release_note", "cypress:run-without-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=false", "cypress:run-with-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=true,openSearchUrl=https://localhost:9200,WAIT_FOR_LOADER_BUFFER_MS=500", "osd:ciGroup10": "echo \"dashboard_sanity_test_spec.js\"", diff --git a/scripts/generate_release_note.js b/scripts/generate_release_note.js new file mode 100644 index 000000000000..4721fe0dec35 --- /dev/null +++ b/scripts/generate_release_note.js @@ -0,0 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +require('../src/setup_node_env'); +require('../src/dev/generate_release_note'); +require('../src/dev/generate_release_note_helper'); diff --git a/src/dev/generate_release_note.ts b/src/dev/generate_release_note.ts new file mode 100644 index 000000000000..4c9eaabf0bf7 --- /dev/null +++ b/src/dev/generate_release_note.ts @@ -0,0 +1,134 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { join, resolve } from 'path'; +import { readFileSync, writeFileSync, Dirent, rm, rename, promises as fsPromises } from 'fs'; +import { load as loadYaml } from 'js-yaml'; +import { readdir } from 'fs/promises'; +import { version as pkgVersion } from '../../package.json'; +import { + validateFragment, + getCurrentDateFormatted, + Changelog, + SECTION_MAPPING, + fragmentDirPath, + SectionKey, + releaseNotesDirPath, + filePath, +} from './generate_release_note_helper'; + +// Function to add content after the 'Unreleased' section in the changelog +function addContentAfterUnreleased(path: string, newContent: string): void { + let fileContent = readFileSync(path, 'utf8'); + const targetString = '## [Unreleased]'; + const targetIndex = fileContent.indexOf(targetString); + + if (targetIndex !== -1) { + const endOfLineIndex = fileContent.indexOf('\n', targetIndex); + if (endOfLineIndex !== -1) { + fileContent = + fileContent.slice(0, endOfLineIndex + 1) + + '\n' + + newContent + + '\n' + + fileContent.slice(endOfLineIndex + 1); + } else { + throw new Error('End of line for "Unreleased" section not found.'); + } + } else { + throw new Error("'## [Unreleased]' not found in the file."); + } + + writeFileSync(path, fileContent); +} + +async function deleteFragments(fragmentTempDirPath: string) { + rm(fragmentTempDirPath, { recursive: true }, (err: any) => { + if (err) { + throw err; + } + }); +} + +// Read fragment files and populate sections +async function readFragments() { + // Initialize sections + const sections: Changelog = (Object.fromEntries( + Object.keys(SECTION_MAPPING).map((key) => [key, []]) + ) as unknown) as Changelog; + + const fragmentPaths = await readdir(fragmentDirPath, { withFileTypes: true }); + for (const fragmentFilename of fragmentPaths) { + // skip non yml or yaml files + if (!/\.ya?ml$/i.test(fragmentFilename.name)) { + // eslint-disable-next-line no-console + console.warn(`Skipping non yml or yaml file ${fragmentFilename.name}`); + continue; + } + + const fragmentPath = join(fragmentDirPath, fragmentFilename.name); + const fragmentContents = readFileSync(fragmentPath, { encoding: 'utf-8' }); + + validateFragment(fragmentContents); + + const fragmentYaml = loadYaml(fragmentContents) as Changelog; + + for (const [sectionKey, entries] of Object.entries(fragmentYaml)) { + sections[sectionKey as SectionKey].push(...entries); + } + } + return { sections, fragmentPaths }; +} + +async function moveFragments(fragmentPaths: Dirent[], fragmentTempDirPath: string): Promise { + // Move fragment files to temp fragments folder + for (const fragmentFilename of fragmentPaths) { + const fragmentPath = resolve(fragmentDirPath, fragmentFilename.name); + const fragmentTempPath = resolve(fragmentTempDirPath, fragmentFilename.name); + rename(fragmentPath, fragmentTempPath, () => {}); + } +} + +function generateChangelog(sections: Changelog) { + // Generate changelog sections + const changelogSections = Object.entries(sections).map(([sectionKey, entries]) => { + const sectionName = SECTION_MAPPING[sectionKey as SectionKey]; + return entries.length === 0 + ? `### ${sectionName}` + : `### ${sectionName}\n\n${entries.map((entry) => ` - ${entry}`).join('\n')}`; + }); + + // Generate full changelog + const currentDate = getCurrentDateFormatted(); + const changelog = `## [${pkgVersion}-${currentDate}](https://github.com/opensearch-project/OpenSearch-Dashboards/releases/tag/${pkgVersion})\n\n${changelogSections.join( + '\n\n' + )}`; + // Update changelog file + addContentAfterUnreleased(filePath, changelog); + return changelogSections; +} + +function generateReleaseNote(changelogSections: string[]) { + // Generate release note + const releaseNoteFilename = `opensearch-dashboards.release-notes-${pkgVersion}.md`; + const releaseNoteHeader = `# VERSION ${pkgVersion} Release Note`; + const releaseNote = `${releaseNoteHeader}\n\n${changelogSections.join('\n\n')}`; + writeFileSync(resolve(releaseNotesDirPath, releaseNoteFilename), releaseNote); +} + +(async () => { + const { sections, fragmentPaths } = await readFragments(); + // create folder for temp fragments + const fragmentTempDirPath = await fsPromises.mkdtemp(join(fragmentDirPath, 'tmp_fragments-')); + // move fragments to temp fragments folder + await moveFragments(fragmentPaths, fragmentTempDirPath); + + const changelogSections = generateChangelog(sections); + + generateReleaseNote(changelogSections); + + // remove temp fragments folder + await deleteFragments(fragmentTempDirPath); +})(); diff --git a/src/dev/generate_release_note_helper.ts b/src/dev/generate_release_note_helper.ts new file mode 100644 index 000000000000..988c0f92d964 --- /dev/null +++ b/src/dev/generate_release_note_helper.ts @@ -0,0 +1,59 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { resolve } from 'path'; + +export const filePath = resolve(__dirname, '..', '..', 'CHANGELOG.md'); +export const fragmentDirPath = resolve(__dirname, '..', '..', 'changelogs', 'fragments'); +export const releaseNotesDirPath = resolve(__dirname, '..', '..', 'release-notes'); + +export function getCurrentDateFormatted(): string { + return new Date().toISOString().slice(0, 10); +} + +export const SECTION_MAPPING = { + breaking: '💥 Breaking Changes', + deprecate: 'Deprecations', + security: '🛡 Security', + feat: '📈 Features/Enhancements', + fix: '🐛 Bug Fixes', + infra: '🚞 Infrastructure', + doc: '📝 Documentation', + chore: '🛠 Maintenance', + refactor: '🪛 Refactoring', + test: '🔩 Tests', +}; + +export type SectionKey = keyof typeof SECTION_MAPPING; +export type Changelog = Record; + +const MAX_ENTRY_LENGTH = 100; +// Each entry must start with '-' and a space, followed by a non-empty string, and be no longer that MAX_ENTRY_LENGTH characters +const entryRegex = new RegExp(`^-.{1,${MAX_ENTRY_LENGTH}}\\(\\[#.+]\\(.+\\)\\)$`); + +// validate format of fragment files +export function validateFragment(content: string) { + const sections = content.split(/(?:\r?\n){2,}/); + + // validate each section + for (const section of sections) { + const lines = section.split('\n'); + const sectionName = lines[0]; + const sectionKey = sectionName.slice(0, -1); + + if (!SECTION_MAPPING[sectionKey as SectionKey] || !sectionName.endsWith(':')) { + throw new Error(`Unknown section ${sectionKey}.`); + } + for (const entry of lines.slice(1)) { + if (entry === '') { + continue; + } + // if (!entryRegex.test(entry)) { + if (!entryRegex.test(entry.trim())) { + throw new Error(`Invalid entry ${entry} in section ${sectionKey}.`); + } + } + } +}