ci: implement new release flow

Signed-off-by: Mohammed Al Sahaf <msaa1990@gmail.com>

.github/workflows/release-proposal.yml

@@ -0,0 +1,157 @@
+name: Release Proposal
+
+# This workflow creates a release proposal that requires approval from maintainers
+# Triggered manually by maintainers when ready to prepare a release
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to release (e.g., v2.8.0)'
+        required: true
+        type: string
+      prerelease:
+        description: 'Is this a pre-release?'
+        required: false
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+  issues: write
+  pull-requests: write
+
+jobs:
+  create-proposal:
+    name: Create Release Proposal
+    runs-on: ubuntu-latest
+    
+    steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1
+        with:
+          egress-policy: audit
+      - name: Checkout code
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          fetch-depth: 0
+
+      - name: Validate version format
+        run: |
+          if [[ ! "${{ inputs.version }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$ ]]; then
+            echo "Error: Version must follow semver format (e.g., v2.8.0 or v2.8.0-beta.1)"
+            exit 1
+          fi
+
+      - name: Check if tag already exists
+        run: |
+          if git rev-parse "${{ inputs.version }}" >/dev/null 2>&1; then
+            echo "Error: Tag ${{ inputs.version }} already exists"
+            exit 1
+          fi
+
+      - name: Generate changelog
+        id: changelog
+        run: |
+          # Get the HEAD commit hash
+          HEAD_COMMIT=$(git rev-parse HEAD)
+          echo "head_commit=$HEAD_COMMIT" >> $GITHUB_OUTPUT
+          
+          # Get the last tag
+          LAST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "")
+          
+          if [ -z "$LAST_TAG" ]; then
+            echo "No previous tag found, generating full changelog"
+            COMMITS=$(git log --pretty=format:"- %s (%h)" --reverse)
+          else
+            echo "Generating changelog since $LAST_TAG"
+            COMMITS=$(git log ${LAST_TAG}..HEAD --pretty=format:"- %s (%h)" --reverse)
+          fi
+          
+          # Save commits to file
+          echo "$COMMITS" > /tmp/commits.txt
+          
+          echo "commits_file=/tmp/commits.txt" >> $GITHUB_OUTPUT
+
+      - name: Create release proposal issue
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const fs = require('fs');
+            const commits = fs.readFileSync('/tmp/commits.txt', 'utf8');
+            const isPrerelease = '${{ inputs.prerelease }}' === 'true';
+            const releaseType = isPrerelease ? 'Pre-release' : 'Stable Release';
+            
+            const body = [
+              '## Release Proposal: ${{ inputs.version }}',
+              '',
+              '**Proposed by:** @${{ github.actor }}',
+              `**Type:** ${releaseType}`,
+              '**Commit:** `${{ steps.changelog.outputs.head_commit }}`',
+              '',
+              '### Approval Requirements',
+              '',
+              '- [ ] Minimum 2 maintainer approvals required (use 👍 reaction)',
+              '- [ ] CI/CD checks must pass',
+              '- [ ] Security review completed',
+              '- [ ] Documentation updated',
+              '',
+              '### Changes Since Last Release',
+              '',
+              commits.trim(),
+              '',
+              '### Next Steps',
+              '',
+              '1. Review the changes above',
+              '2. Verify all tests pass',
+              '3. Maintainers: React with 👍 to approve',
+              '4. Once approved, a maintainer will run the tag creation workflow',
+              '',
+              '### Maintainer Actions',
+              '',
+              'After approval, run:',
+              '```bash',
+              '# Pull latest changes',
+              'git checkout master',
+              'git pull origin master',
+              '',
+              '# Create signed tag (requires SSH key)',
+              'git tag -s -m "Release ${{ inputs.version }}" ${{ inputs.version }} ${{ steps.changelog.outputs.head_commit }}',
+              '',
+              '# Verify the tag signature',
+              'git tag -v ${{ inputs.version }}',
+              '',
+              '# Push the tag',
+              'git push origin ${{ inputs.version }}',
+              '```',
+              '',
+              '---',
+              '',
+              '**Automation Note:** This proposal requires manual tag creation to ensure maintainer signatures are used.'
+            ].join('\n');
+            
+            const issue = await github.rest.issues.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              title: `Release Proposal: ${{ inputs.version }}`,
+              body: body,
+              labels: ['release-proposal', 'needs-approval']
+            });
+            
+            console.log(`Created issue: ${issue.data.html_url}`);
+            
+            // Pin the issue
+            await github.rest.issues.update({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: issue.data.number,
+              state: 'open'
+            });
+
+      - name: Post summary
+        run: |
+          echo "## Release Proposal Created! 🚀" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "Version: **${{ inputs.version }}**" >> $GITHUB_STEP_SUMMARY
+          echo "Type: **${{ inputs.prerelease == 'true' && 'Pre-release' || 'Stable Release' }}**" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "Check the issues tab for the release proposal." >> $GITHUB_STEP_SUMMARY

.github/workflows/release.yml

@@ -13,8 +13,291 @@ permissions:
   contents: read
 
 jobs:
+  verify-tag:
+    name: Verify Tag Signature and Approvals
+    runs-on: ubuntu-latest
+    # environment: 'default'
+    
+    outputs:
+      verification_passed: ${{ steps.verify.outputs.passed }}
+      tag_version: ${{ steps.info.outputs.version }}
+      proposal_issue_number: ${{ steps.find_proposal.outputs.result && fromJson(steps.find_proposal.outputs.result).number || '' }}
+    
+    steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1
+        with:
+          egress-policy: audit
+      - name: Checkout code
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          fetch-depth: 0
+
+      - name: Get tag info
+        id: info
+        run: |
+          echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+          echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+
+       # https://github.community/t5/GitHub-Actions/How-to-get-just-the-tag-name/m-p/32167/highlight/true#M1027
+      - name: Print Go version and environment
+        id: vars
+        run: |
+          printf "Using go at: $(which go)\n"
+          printf "Go version: $(go version)\n"
+          printf "\n\nGo environment:\n\n"
+          go env
+          printf "\n\nSystem environment:\n\n"
+          env
+          echo "::set-output name=version_tag::${GITHUB_REF/refs\/tags\//}"
+          echo "::set-output name=short_sha::$(git rev-parse --short HEAD)"
+
+          # Add "pip install" CLI tools to PATH
+          echo ~/.local/bin >> $GITHUB_PATH
+
+          # Parse semver
+          TAG=${GITHUB_REF/refs\/tags\//}
+          SEMVER_RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z\.-]*\)'
+          TAG_MAJOR=`echo ${TAG#v} | sed -e "s#$SEMVER_RE#\1#"`
+          TAG_MINOR=`echo ${TAG#v} | sed -e "s#$SEMVER_RE#\2#"`
+          TAG_PATCH=`echo ${TAG#v} | sed -e "s#$SEMVER_RE#\3#"`
+          TAG_SPECIAL=`echo ${TAG#v} | sed -e "s#$SEMVER_RE#\4#"`
+          echo "::set-output name=tag_major::${TAG_MAJOR}"
+          echo "::set-output name=tag_minor::${TAG_MINOR}"
+          echo "::set-output name=tag_patch::${TAG_PATCH}"
+          echo "::set-output name=tag_special::${TAG_SPECIAL}"
+
+      - name: Validate commits and tag signatures
+        id: verify
+        env:
+          signing_keys: ${{ secrets.SIGNING_KEYS }}
+        run: |
+          # Read the string into an array, splitting by IFS
+          IFS=";" read -ra keys_collection <<< "$signing_keys"
+          
+          # ref: https://docs.github.com/en/actions/reference/workflows-and-actions/contexts#example-usage-of-the-runner-context
+          touch "${{ runner.temp }}/allowed_signers"
+
+          # Iterate and print the split elements
+          for item in "${keys_collection[@]}"; do
+          
+            # trim leading whitespaces
+            item="${item##*( )}"
+
+            # trim trailing whitespaces
+            item="${item%%*( )}"
+            
+            IFS=" " read -ra key_components <<< "$item"
+            # email address, type, public key
+            echo "${key_components[0]} namespaces=\"git\" ${key_components[1]} ${key_components[2]}" >> "${{ runner.temp }}/allowed_signers"
+          done
+
+          git config --global gpg.ssh.allowedSignersFile "${{ runner.temp }}/allowed_signers"
+
+          echo "Verifying the tag: ${{ steps.vars.outputs.version_tag }}"
+          
+          # Verify the tag is signed
+          if ! git verify-tag -v "${{ steps.vars.outputs.version_tag }}" 2>&1 | tee /tmp/verify-output.txt; then
+            echo "❌ Tag verification failed!"
+            echo "passed=false" >> $GITHUB_OUTPUT
+            git push --delete origin "${{ steps.vars.outputs.version_tag }}"
+            exit 1
+          fi
+          echo "✅ Tag verification succeeded!"
+          echo "passed=true" >> $GITHUB_OUTPUT
+
+      - name: Find related release proposal
+        id: find_proposal
+        uses: actions/github-script@v8
+        env:
+          MAINTAINER_LOGINS: ${{ secrets.MAINTAINER_LOGINS }}
+        with:
+          script: |
+            const version = '${{ steps.vars.outputs.version_tag }}';
+            
+            // Search for the release proposal issue
+            const issues = await github.rest.issues.listForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              labels: 'release-proposal',
+              state: 'open',
+              labels: 'release-proposal,needs-approval'
+            });
+            
+            const proposal = issues.data.find(issue => 
+              issue.title.includes(version)
+            );
+            
+            if (!proposal) {
+              console.log(`⚠️  No release proposal found for ${version}`);
+              console.log('This might be a hotfix or emergency release');
+              return { number: null, approved: false, approvals: 0 };
+            }
+            
+            // Get reactions to check for approvals (👍 emoji)
+            const reactions = await github.rest.reactions.listForIssue({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: proposal.number,
+              content: '+1'
+            });
+
+            console.log(`Found proposal #${proposal.number} for version ${version} with ${reactions.data.length} 👍 reactions`);
+            
+            // Extract commit hash from proposal body
+            const commitMatch = proposal.body.match(/\*\*Commit:\*\*\s*`([a-f0-9]+)`/);
+            const proposedCommit = commitMatch ? commitMatch[1] : null;
+            
+            if (proposedCommit) {
+              console.log(`Proposal is for commit: ${proposedCommit}`);
+            } else {
+              console.log('⚠️  No commit hash found in proposal');
+            }
+            
+            // Get maintainer logins from secret (comma or semicolon separated)
+            const maintainerLoginsStr = process.env.MAINTAINER_LOGINS || '';
+            const maintainerLogins = new Set(
+              maintainerLoginsStr
+                .split(/[,;]/)
+                .map(login => login.trim().toLowerCase())
+                .filter(login => login.length > 0)
+            );
+            
+            console.log(`Found ${maintainerLogins.size} maintainer logins in secret`);
+            
+            // Count approvals from maintainers
+            const maintainerApprovals = reactions.data.filter(reaction => 
+              maintainerLogins.has(reaction.user.login.toLowerCase())
+            );
+            
+            const approvalCount = maintainerApprovals.length;
+            const approved = approvalCount >= 2;
+            
+            console.log(`Found ${approvalCount} maintainer approvals`);
+            console.log(`Approved: ${approved}`);
+            
+            return {
+              number: proposal.number,
+              approved: approved,
+              approvals: approvalCount,
+              approvers: maintainerApprovals.map(r => '@' + r.user.login).join(', '),
+              proposedCommit: proposedCommit
+            };
+          result-encoding: json
+
+      - name: Check approval requirements
+        run: |
+          APPROVALS='${{ steps.find_proposal.outputs.result }}'
+          
+          # Parse JSON
+          APPROVED=$(echo "$APPROVALS" | jq -r '.approved')
+          COUNT=$(echo "$APPROVALS" | jq -r '.approvals')
+          APPROVERS=$(echo "$APPROVALS" | jq -r '.approvers')
+          PROPOSED_COMMIT=$(echo "$APPROVALS" | jq -r '.proposedCommit')
+          CURRENT_COMMIT="${{ steps.info.outputs.sha }}"
+          
+          echo "Approval status: $APPROVED"
+          echo "Approval count: $COUNT"
+          echo "Approvers: $APPROVERS"
+          echo "Proposed commit: $PROPOSED_COMMIT"
+          echo "Current commit: $CURRENT_COMMIT"
+          
+          # Check if commits match
+          if [ "$PROPOSED_COMMIT" != "null" ] && [ "$PROPOSED_COMMIT" != "$CURRENT_COMMIT" ]; then
+            echo "❌ Commit mismatch!"
+            echo "The tag points to commit $CURRENT_COMMIT but the proposal was for $PROPOSED_COMMIT"
+            echo "This indicates the code has changed since the proposal was created."
+            # delete the tag
+            git push --delete origin "${{ steps.vars.outputs.version_tag }}"
+            echo "Tag ${{steps.vars.outputs.version_tag}} has been deleted"
+            exit 1
+          fi
+          
+          if [ "$PROPOSED_COMMIT" != "null" ]; then
+            echo "✅ Commit hash matches proposal"
+          fi
+          
+          if [ "$APPROVED" != "true" ]; then
+            echo "❌ Release does not have minimum 2 maintainer approvals"
+            echo "Current approvals: $COUNT"
+            # Delete the tag remotely
+            git push --delete origin "${{ steps.vars.outputs.version_tag }}"
+            
+            echo "Tag ${{steps.vars.outputs.version_tag}} has been deleted"
+            exit 1
+          fi
+          
+          echo "✅ Release has sufficient approvals"
+
+      - name: Update release proposal
+        if: steps.find_proposal.outputs.result != 'null'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const result = ${{ steps.find_proposal.outputs.result }};
+            
+            if (result.number) {
+              const commentBody = [
+                '## ✅ Release Tag Created and Verified',
+                '',
+                '- **Tag:** ${{ steps.info.outputs.version }}',
+                '- **Signed by key:** ${{ steps.verify.outputs.key_id }}',
+                `- **Approvals:** ${result.approvals} maintainers (${result.approvers})`,
+                '- **Commit:** ${{ steps.info.outputs.sha }}',
+                '',
+                'Release workflow is now running. This issue will be closed when the release is published.'
+              ].join('\n');
+              
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: result.number,
+                body: commentBody
+              });
+
+              // remove earlier labels
+              await github.rest.issues.removeLabel({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: result.number,
+                name: 'needs-approval'
+              });
+              await github.rest.issues.removeLabel({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: result.number,
+                name: 'release-proposal'
+              });
+              
+              // add 'release-in-progress' label
+              await github.rest.issues.addLabels({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: result.number,
+                labels: ['release-in-progress']
+              });
+            }
+
+      - name: Summary
+        run: |
+          APPROVALS='${{ steps.find_proposal.outputs.result }}'
+          PROPOSED_COMMIT=$(echo "$APPROVALS" | jq -r '.proposedCommit // "N/A"')
+          
+          echo "## Tag Verification Summary 🔐" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "- **Tag:** ${{ steps.info.outputs.version }}" >> $GITHUB_STEP_SUMMARY
+          echo "- **Commit:** ${{ steps.info.outputs.sha }}" >> $GITHUB_STEP_SUMMARY
+          echo "- **Proposed Commit:** $PROPOSED_COMMIT" >> $GITHUB_STEP_SUMMARY
+          echo "- **Signature:** ✅ Verified" >> $GITHUB_STEP_SUMMARY
+          echo "- **Signed by:** ${{ steps.verify.outputs.key_id }}" >> $GITHUB_STEP_SUMMARY
+          echo "- **Approvals:** ✅ Sufficient" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "Proceeding with release build..." >> $GITHUB_STEP_SUMMARY
+      
   release:
     name: Release
+    needs: verify-tag
+    if: ${{ needs.verify-tag.outputs.verification_passed == 'true' }}
     strategy:
       matrix:
         os: 
@@ -36,6 +319,7 @@ jobs:
       # https://docs.github.com/en/rest/overview/permissions-required-for-github-apps#permission-on-contents
       # "Releases" is part of `contents`, so it needs the `write`
       contents: write
+      issues: write
 
     steps:
     - name: Harden the runner (Audit all outbound calls)
@@ -99,13 +383,34 @@ jobs:
       run: pip install --upgrade cloudsmith-cli
 
     - name: Validate commits and tag signatures
+      env:
+        signing_keys: ${{ secrets.SIGNING_KEYS }}
       run: |
+        # Read the string into an array, splitting by IFS
+        IFS=";" read -ra keys_collection <<< "$signing_keys"
         
-        # Import Matt Holt's key
+        # ref: https://docs.github.com/en/actions/reference/workflows-and-actions/contexts#example-usage-of-the-runner-context
-        curl 'https://github.com/mholt.gpg' | gpg --import
+        touch "${{ runner.temp }}/allowed_signers"
+
+        # Iterate and print the split elements
+        for item in "${keys_collection[@]}"; do
+        
+          # trim leading whitespaces
+          item="${item##*( )}"
+
+          # trim trailing whitespaces
+          item="${item%%*( )}"
+          
+          IFS=" " read -ra key_components <<< "$item"
+          # [email address] [type] [public key]
+          echo "${key_components[0]} namespaces=\"git\" ${key_components[1]} ${key_components[2]}" >> "${{ runner.temp }}/allowed_signers"
+        done
+
+        git config --global gpg.ssh.allowedSignersFile "${{ runner.temp }}/allowed_signers"
 
         echo "Verifying the tag: ${{ steps.vars.outputs.version_tag }}"
-        # tags are only accepted if signed by Matt's key
+        
+        # tags are only accepted if signed by a trusted key
         git verify-tag "${{ steps.vars.outputs.version_tag }}" || exit 1
 
     - name: Install Cosign
@@ -188,3 +493,42 @@ jobs:
           echo "Pushing $filename to 'testing'"
           cloudsmith push deb caddy/testing/any-distro/any-version $filename
         done
+        
+    - name: Close release proposal issue
+      if: needs.verify-tag.outputs.proposal_issue_number != ''
+      uses: actions/github-script@v7
+      with:
+        script: |
+          const issueNumber = parseInt('${{ needs.verify-tag.outputs.proposal_issue_number }}');
+          
+          if (issueNumber) {
+            // Add final comment
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: issueNumber,
+              body: '## ✅ Release Published\n\nThe release has been successfully published. Closing this proposal.'
+            });
+            
+            // Remove the release-in-progress label
+            try {
+              await github.rest.issues.removeAllLabels({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: issueNumber
+              });
+            } catch (error) {
+              console.log('Label might not exist on issue');
+            }
+            
+            // Close the issue
+            await github.rest.issues.update({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: issueNumber,
+              state: 'closed',
+              state_reason: 'completed'
+            });
+            
+            console.log(`Closed issue #${issueNumber}`);
+          }