Mirrors/yet-another-bench-script
1
0
Fork 0
mirror of https://github.com/masonr/yet-another-bench-script.git synced 2026-10-07 14:06:38 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

update fio/iperf3 binary generation workflows

Browse Source
This commit is contained in:
Mason Rowe
2026-09-19 20:31:32 -04:00
parent bf41ddf401
commit c0aa5b8870
3 changed files with 189 additions and 260 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build-fio.yml
+98 -95
View File
@@ -2,32 +2,40 @@ name: Build fio Static Binaries
on: on:
workflow_dispatch: workflow_dispatch:
schedule:
# check upstream for a new fio release once a day
- cron: '23 5 * * *'
permissions:
contents: write
jobs: jobs:
check-release: check-release:
runs-on: self-hosted runs-on: ubuntu-latest
outputs: outputs:
latest-version: ${{ steps.get-version.outputs.version }} latest-version: ${{ steps.get-version.outputs.version }}
should-build: ${{ steps.check-version.outputs.should-build }} should-build: ${{ steps.check-version.outputs.should-build }}
steps: steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Get latest fio release - name: Get latest fio release
id: get-version id: get-version
run: | run: |
# Get the latest release tag from GitHub API # Get the latest release tag from GitHub API (e.g. "fio-3.41")
LATEST_VERSION=$(curl -4 --retry 5 --retry-delay 2 --connect-timeout 15 -s https://api.github.com/repos/axboe/fio/releases/latest | jq -r '.tag_name') LATEST_VERSION=$(curl -4 --retry 5 --retry-delay 2 --connect-timeout 15 -s https://api.github.com/repos/axboe/fio/releases/latest | jq -r '.tag_name')
echo "Latest fio version: $LATEST_VERSION" echo "Latest fio version: $LATEST_VERSION"
if [ -z "$LATEST_VERSION" ] || [ "$LATEST_VERSION" = "null" ]; then
echo "Failed to determine latest fio version"
exit 1
fi
echo "version=$LATEST_VERSION" >> $GITHUB_OUTPUT echo "version=$LATEST_VERSION" >> $GITHUB_OUTPUT
- name: Check if version exists in releases - name: Check if version exists in releases
id: check-version id: check-version
run: | run: |
VERSION="${{ steps.get-version.outputs.version }}" VERSION="${{ steps.get-version.outputs.version }}"
# Check if this version already exists in our releases # Check if this version already exists in our releases (upstream tag is
if gh release view "fio-$VERSION" --repo ${{ github.repository }} >/dev/null 2>&1; then # already prefixed, e.g. "fio-3.41", so it can be used as-is)
if gh release view "$VERSION" --repo ${{ github.repository }} >/dev/null 2>&1; then
echo "Version $VERSION already exists in releases" echo "Version $VERSION already exists in releases"
echo "should-build=false" >> $GITHUB_OUTPUT echo "should-build=false" >> $GITHUB_OUTPUT
else else
@@ -40,8 +48,9 @@ jobs:
build: build:
needs: check-release needs: check-release
if: needs.check-release.outputs.should-build == 'true' if: needs.check-release.outputs.should-build == 'true'
runs-on: self-hosted runs-on: ubuntu-latest
strategy: strategy:
fail-fast: false
matrix: matrix:
include: include:
- arch: x64 - arch: x64
@@ -58,81 +67,69 @@ jobs:
host: arm-linux-gnueabihf host: arm-linux-gnueabihf
steps: steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Create compilation script - name: Create compilation script
run: | run: |
VERSION="${{ needs.check-release.outputs.latest-version }}"
ARCH="${{ matrix.arch }}"
CROSS="${{ matrix.cross }}"
HOST="${{ matrix.host }}"
# Single script for all architectures using musl cross-compilation # Single script for all architectures using musl cross-compilation
cat > compile-fio.sh << EOF cat > compile-fio.sh << 'EOF'
#!/bin/bash #!/bin/bash
set -e set -e
# Activate Holy Build Box lib compilation environment # Activate Holy Build Box lib compilation environment
source /hbb/activate source /hbb/activate
set -x set -x
# remove obsolete CentOS repos # remove obsolete CentOS repos
cd /etc/yum.repos.d/ cd /etc/yum.repos.d/
rm -f CentOS-Base.repo CentOS-SCLo-scl-rh.repo CentOS-SCLo-scl.repo CentOS-fasttrack.repo CentOS-x86_64-kernel.repo rm -f CentOS-Base.repo CentOS-SCLo-scl-rh.repo CentOS-SCLo-scl.repo CentOS-fasttrack.repo CentOS-x86_64-kernel.repo
yum install -y yum-plugin-ovl # fix for docker overlay fs yum install -y yum-plugin-ovl # fix for docker overlay fs
yum install -y xz yum install -y xz
# download musl cross compilation toolchain # download musl cross compilation toolchain
cd ~ cd ~
curl -L -4 --retry 5 --retry-delay 2 --connect-timeout 15 "https://musl.cc/\$CROSS-cross.tgz" -o "\$CROSS-cross.tgz" curl -L -4 --retry 5 --retry-delay 2 --connect-timeout 15 "https://musl.cc/$CROSS-cross.tgz" -o "$CROSS-cross.tgz"
tar xf "\$CROSS-cross.tgz" tar xf "$CROSS-cross.tgz"
# download, compile, and install libaio as static library # download, compile, and install libaio as static library
cd ~ cd ~
curl -L -4 --retry 5 --retry-delay 2 --connect-timeout 15 http://ftp.de.debian.org/debian/pool/main/liba/libaio/libaio_0.3.113.orig.tar.gz -o "libaio.tar.gz" curl -L -4 --retry 5 --retry-delay 2 --connect-timeout 15 http://ftp.de.debian.org/debian/pool/main/liba/libaio/libaio_0.3.113.orig.tar.gz -o "libaio.tar.gz"
tar xf libaio.tar.gz tar xf libaio.tar.gz
cd libaio-*/src cd libaio-*/src
CC=/root/\$CROSS-cross/bin/\$CROSS-gcc ENABLE_SHARED=0 make prefix=/hbb_exe install CC=/root/$CROSS-cross/bin/$CROSS-gcc ENABLE_SHARED=0 make prefix=/hbb_exe install
# Activate Holy Build Box exe compilation environment # Activate Holy Build Box exe compilation environment
source /hbb_exe/activate source /hbb_exe/activate
# download and compile fio # download and compile fio
cd ~ cd ~
curl -L -4 --retry 5 --retry-delay 2 --connect-timeout 15 "https://github.com/axboe/fio/archive/\$VERSION.tar.gz" -o "fio.tar.gz" curl -L -4 --retry 5 --retry-delay 2 --connect-timeout 15 "https://github.com/axboe/fio/archive/$VERSION.tar.gz" -o "fio.tar.gz"
tar xf fio.tar.gz tar xf fio.tar.gz
cd fio-\${VERSION#fio-}* cd fio-fio-*
CC=/root/\$CROSS-cross/bin/\$CROSS-gcc ./configure --disable-native --build-static # fio >= 3.42 includes both linux/prctl.h and sys/prctl.h in backend.c, which
make # redefines struct prctl_mm_map under musl; sys/prctl.h alone is sufficient
sed -i '/#include <linux\/prctl.h>/d' backend.c
CC=/root/$CROSS-cross/bin/$CROSS-gcc ./configure --disable-native --build-static
# link against libatomic for 32-bit/ARM targets that lack native 64-bit atomics
make EXTLIBS+=' -latomic'
# verify no external shared library links # verify no external shared library links
libcheck fio libcheck fio
# copy fio binary to mounted dir # copy fio binary to mounted dir
cp fio "/io/fio_\$ARCH" cp fio "/io/fio_$ARCH"
EOF EOF
chmod +x compile-fio.sh chmod +x compile-fio.sh
- name: Compile fio binary - name: Compile fio binary
run: | run: |
ARCH="${{ matrix.arch }}"
CROSS="${{ matrix.cross }}"
HOST="${{ matrix.host }}"
# Use musl cross-compilation for all architectures # Use musl cross-compilation for all architectures
docker run --rm -v $(pwd):/io --env ARCH=$ARCH --env CROSS=$CROSS --env HOST=$HOST --env VERSION="${{ needs.check-release.outputs.latest-version }}" phusion/holy-build-box-64:latest bash /io/compile-fio.sh docker run --rm -v $(pwd):/io --env ARCH=${{ matrix.arch }} --env CROSS=${{ matrix.cross }} --env HOST=${{ matrix.host }} --env VERSION="${{ needs.check-release.outputs.latest-version }}" phusion/holy-build-box-64:latest bash /io/compile-fio.sh
- name: Verify binary - name: Verify binary
run: | run: |
ARCH="${{ matrix.arch }}" ls -la fio_${{ matrix.arch }}
ls -la fio_$ARCH file fio_${{ matrix.arch }}
file fio_$ARCH
- name: Upload binary as artifact - name: Upload binary as artifact
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
@@ -144,7 +141,7 @@ jobs:
virustotal-scan: virustotal-scan:
needs: [check-release, build] needs: [check-release, build]
if: needs.check-release.outputs.should-build == 'true' if: needs.check-release.outputs.should-build == 'true'
runs-on: self-hosted runs-on: ubuntu-latest
outputs: outputs:
scan-results: ${{ steps.scan-summary.outputs.results }} scan-results: ${{ steps.scan-summary.outputs.results }}
all-clean: ${{ steps.scan-summary.outputs.all-clean }} all-clean: ${{ steps.scan-summary.outputs.all-clean }}
@@ -166,14 +163,13 @@ jobs:
echo "## VirusTotal Scan Results" > scan_results.md echo "## VirusTotal Scan Results" > scan_results.md
echo "| Binary | Status | Malicious | Suspicious | Undetected | VirusTotal URL |" >> scan_results.md echo "| Binary | Status | Malicious | Suspicious | Undetected | VirusTotal URL |" >> scan_results.md
echo "|--------|--------|-----------|------------|------------|----------------|" >> scan_results.md echo "|--------|--------|-----------|------------|------------|----------------|" >> scan_results.md
ALL_CLEAN=true ALL_CLEAN=true
SCAN_DATA=""
for binary in scan-binaries/fio_*; do for binary in scan-binaries/fio_*; do
filename=$(basename "$binary") filename=$(basename "$binary")
echo "Uploading $filename to VirusTotal..." echo "Uploading $filename to VirusTotal..."
# Upload to VirusTotal # Upload to VirusTotal
upload_response=$(curl -4 --retry 5 --retry-delay 2 --connect-timeout 15 -s --request POST \ upload_response=$(curl -4 --retry 5 --retry-delay 2 --connect-timeout 15 -s --request POST \
--url https://www.virustotal.com/api/v3/files \ --url https://www.virustotal.com/api/v3/files \
@@ -181,23 +177,21 @@ jobs:
--header 'content-type: multipart/form-data' \ --header 'content-type: multipart/form-data' \
--header "x-apikey: ${{ secrets.VIRUSTOTAL_API_KEY }}" \ --header "x-apikey: ${{ secrets.VIRUSTOTAL_API_KEY }}" \
--form "file=@$binary") --form "file=@$binary")
analysis_id=$(echo "$upload_response" | jq -r '.data.id') analysis_id=$(echo "$upload_response" | jq -r '.data.id')
echo "Analysis ID for $filename: $analysis_id" echo "Analysis ID for $filename: $analysis_id"
if [ "$analysis_id" = "null" ] || [ -z "$analysis_id" ]; then if [ "$analysis_id" = "null" ] || [ -z "$analysis_id" ]; then
echo "Failed to upload $filename to VirusTotal" echo "Failed to upload $filename to VirusTotal"
echo "| $filename | Upload Failed | N/A | N/A | N/A | N/A |" >> scan_results.md echo "| $filename | Upload Failed | N/A | N/A | N/A | N/A |" >> scan_results.md
ALL_CLEAN=false ALL_CLEAN=false
continue continue
fi fi
# Store analysis ID for later retrieval # Store analysis ID for later retrieval
echo "${filename}:${analysis_id}" >> analysis_ids.txt echo "${filename}:${analysis_id}" >> analysis_ids.txt
done done
echo "all_clean_upload=$ALL_CLEAN" >> $GITHUB_OUTPUT
# Wait 2 minutes for scans to complete # Wait 2 minutes for scans to complete
echo "Waiting 2 minutes for VirusTotal scans to complete..." echo "Waiting 2 minutes for VirusTotal scans to complete..."
sleep 120 sleep 120
@@ -206,60 +200,61 @@ jobs:
id: get-results id: get-results
run: | run: |
ALL_CLEAN=true ALL_CLEAN=true
SCAN_SUMMARY=""
while IFS=':' read -r filename analysis_id; do while IFS=':' read -r filename analysis_id; do
echo "Retrieving results for $filename (ID: $analysis_id)..." echo "Retrieving results for $filename (ID: $analysis_id)..."
# Get scan results # Get scan results
result_response=$(curl -4 --retry 5 --retry-delay 2 --connect-timeout 15 -s --request GET \ result_response=$(curl -4 --retry 5 --retry-delay 2 --connect-timeout 15 -s --request GET \
--url "https://www.virustotal.com/api/v3/analyses/$analysis_id" \ --url "https://www.virustotal.com/api/v3/analyses/$analysis_id" \
--header 'accept: application/json' \ --header 'accept: application/json' \
--header "x-apikey: ${{ secrets.VIRUSTOTAL_API_KEY }}") --header "x-apikey: ${{ secrets.VIRUSTOTAL_API_KEY }}")
status=$(echo "$result_response" | jq -r '.data.attributes.status // "unknown"') status=$(echo "$result_response" | jq -r '.data.attributes.status // "unknown"')
if [ "$status" = "completed" ]; then if [ "$status" = "completed" ]; then
malicious=$(echo "$result_response" | jq -r '.data.attributes.stats.malicious // 0') malicious=$(echo "$result_response" | jq -r '.data.attributes.stats.malicious // 0')
suspicious=$(echo "$result_response" | jq -r '.data.attributes.stats.suspicious // 0') suspicious=$(echo "$result_response" | jq -r '.data.attributes.stats.suspicious // 0')
undetected=$(echo "$result_response" | jq -r '.data.attributes.stats.undetected // 0') undetected=$(echo "$result_response" | jq -r '.data.attributes.stats.undetected // 0')
# Get file hash from the item link # Get file hash from the item link
item_link=$(echo "$result_response" | jq -r '.data.links.item // ""') item_link=$(echo "$result_response" | jq -r '.data.links.item // ""')
file_hash=$(echo "$item_link" | sed 's/.*files\///') file_hash=$(echo "$item_link" | sed 's/.*files\///')
vt_url="https://www.virustotal.com/gui/file/$file_hash" vt_url="https://www.virustotal.com/gui/file/$file_hash"
if [ "$malicious" -gt 0 ] || [ "$suspicious" -gt 0 ]; then if [ "$malicious" -gt 0 ] || [ "$suspicious" -gt 0 ]; then
status_text="⚠️ FLAGGED" status_text="⚠️ FLAGGED"
ALL_CLEAN=false ALL_CLEAN=false
else else
status_text="✅ Clean" status_text="✅ Clean"
fi fi
echo "| $filename | $status_text | $malicious | $suspicious | $undetected | [$file_hash]($vt_url) |" >> scan_results.md echo "| $filename | $status_text | $malicious | $suspicious | $undetected | [$file_hash]($vt_url) |" >> scan_results.md
else else
echo "| $filename | ⏳ Pending | N/A | N/A | N/A | Scan not completed |" >> scan_results.md echo "| $filename | ⏳ Pending | N/A | N/A | N/A | Scan not completed |" >> scan_results.md
ALL_CLEAN=false ALL_CLEAN=false
fi fi
done < analysis_ids.txt done < analysis_ids.txt
echo "all_clean=$ALL_CLEAN" >> $GITHUB_OUTPUT echo "all_clean=$ALL_CLEAN" >> $GITHUB_OUTPUT
# Add summary to GitHub step summary # Add summary to GitHub step summary
cat scan_results.md >> $GITHUB_STEP_SUMMARY cat scan_results.md >> $GITHUB_STEP_SUMMARY
- name: Create scan summary - name: Create scan summary
id: scan-summary id: scan-summary
run: | run: |
RESULTS=$(cat scan_results.md)
ALL_CLEAN="${{ steps.get-results.outputs.all_clean }}" ALL_CLEAN="${{ steps.get-results.outputs.all_clean }}"
# Escape newlines for GitHub output # Pass the scan results through as a multi-line output (heredoc delimiter
RESULTS_ESCAPED=$(echo "$RESULTS" | sed ':a;N;$!ba;s/\n/\\n/g') # syntax) so the markdown table renders correctly in the release body
{
echo "results=$RESULTS_ESCAPED" >> $GITHUB_OUTPUT echo "results<<RESULTS_EOF"
cat scan_results.md
echo "RESULTS_EOF"
} >> $GITHUB_OUTPUT
echo "all-clean=$ALL_CLEAN" >> $GITHUB_OUTPUT echo "all-clean=$ALL_CLEAN" >> $GITHUB_OUTPUT
- name: Upload scan results as artifact - name: Upload scan results as artifact
@@ -274,11 +269,8 @@ jobs:
create-release: create-release:
needs: [check-release, build, virustotal-scan] needs: [check-release, build, virustotal-scan]
if: needs.check-release.outputs.should-build == 'true' && needs.virustotal-scan.outputs.all-clean == 'true' if: needs.check-release.outputs.should-build == 'true' && needs.virustotal-scan.outputs.all-clean == 'true'
runs-on: self-hosted runs-on: ubuntu-latest
steps: steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Download all artifacts - name: Download all artifacts
uses: actions/download-artifact@v4 uses: actions/download-artifact@v4
with: with:
@@ -291,6 +283,13 @@ jobs:
find artifacts -name "fio_*" -type f -exec cp {} release-assets/ \; find artifacts -name "fio_*" -type f -exec cp {} release-assets/ \;
ls -la release-assets/ ls -la release-assets/
- name: Download upstream source tarball
run: |
# Attach the exact upstream source these binaries were built from
# (fio is GPL-2.0; distributing binaries requires offering the source)
curl -L --retry 5 --retry-delay 2 --connect-timeout 15 "https://github.com/axboe/fio/archive/${{ needs.check-release.outputs.latest-version }}.tar.gz" -o "release-assets/${{ needs.check-release.outputs.latest-version }}.tar.gz"
ls -la release-assets/
- name: Generate SHA256 checksums - name: Generate SHA256 checksums
run: | run: |
cd release-assets cd release-assets
@@ -304,36 +303,40 @@ jobs:
path: vt-results path: vt-results
- name: Create Release - name: Create Release
uses: softprops/action-gh-release@v1 uses: softprops/action-gh-release@v2
with: with:
tag_name: fio-${{ needs.check-release.outputs.latest-version }} tag_name: ${{ needs.check-release.outputs.latest-version }}
name: fio ${{ needs.check-release.outputs.latest-version }} Static Binaries name: ${{ needs.check-release.outputs.latest-version }} Static Binaries
body: | body: |
Static binaries for fio ${{ needs.check-release.outputs.latest-version }} Static binaries for ${{ needs.check-release.outputs.latest-version }}
Built using musl toolchains for maximum compatibility. Built using musl toolchains for maximum compatibility.
**Architectures:** **Architectures:**
- `fio_x64` - x86_64 (64-bit) - `fio_x64` - x86_64 (64-bit)
- `fio_x86` - i686 (32-bit) - `fio_x86` - i686 (32-bit)
- `fio_aarch64` - ARM 64-bit - `fio_aarch64` - ARM 64-bit
- `fio_arm` - ARM 32-bit - `fio_arm` - ARM 32-bit
**Security Verification:** **Security Verification:**
All binaries have been scanned by VirusTotal and verified clean. All binaries have been scanned by VirusTotal and verified clean.
${{ needs.virustotal-scan.outputs.scan-results }} ${{ needs.virustotal-scan.outputs.scan-results }}
**Checksum Verification:** **Checksum Verification:**
```bash ```bash
# Verify checksums # Verify checksums
sha256sum -c fio-checksums.sha256 sha256sum -c fio-checksums.sha256
``` ```
For usage in YABS script, place these binaries in the `bin/fio/` directory. **License & Source:**
fio is licensed under [GPL-2.0](https://github.com/axboe/fio/blob/master/COPYING) (© Jens Axboe).
These are unofficial static builds produced by CI for use by [YABS](https://github.com/masonr/yet-another-bench-script);
the exact upstream source is attached as `${{ needs.check-release.outputs.latest-version }}.tar.gz`.
files: | files: |
release-assets/fio_* release-assets/fio_*
release-assets/fio-checksums.sha256 release-assets/fio-checksums.sha256
release-assets/*.tar.gz
draft: false draft: false
prerelease: false prerelease: false
env: env:
@@ -342,13 +345,13 @@ jobs:
notify-failure: notify-failure:
needs: [check-release, build, virustotal-scan] needs: [check-release, build, virustotal-scan]
if: always() && needs.check-release.outputs.should-build == 'true' && (failure() || needs.virustotal-scan.outputs.all-clean == 'false') if: always() && needs.check-release.outputs.should-build == 'true' && (failure() || needs.virustotal-scan.outputs.all-clean == 'false')
runs-on: self-hosted runs-on: ubuntu-latest
steps: steps:
- name: Send failure notification - name: Send failure notification
run: | run: |
echo "## ⚠️ fio Build Failed" >> $GITHUB_STEP_SUMMARY echo "## ⚠️ fio Build Failed" >> $GITHUB_STEP_SUMMARY
echo "Version: ${{ needs.check-release.outputs.latest-version }}" >> $GITHUB_STEP_SUMMARY echo "Version: ${{ needs.check-release.outputs.latest-version }}" >> $GITHUB_STEP_SUMMARY
if [ "${{ needs.virustotal-scan.outputs.all-clean }}" == "false" ]; then if [ "${{ needs.virustotal-scan.outputs.all-clean }}" == "false" ]; then
echo "**Reason:** VirusTotal scan detected issues with one or more binaries" >> $GITHUB_STEP_SUMMARY echo "**Reason:** VirusTotal scan detected issues with one or more binaries" >> $GITHUB_STEP_SUMMARY
echo "**Scan Results:**" >> $GITHUB_STEP_SUMMARY echo "**Scan Results:**" >> $GITHUB_STEP_SUMMARY
@@ -356,5 +359,5 @@ jobs:
else else
echo "**Reason:** Build compilation failed" >> $GITHUB_STEP_SUMMARY echo "**Reason:** Build compilation failed" >> $GITHUB_STEP_SUMMARY
fi fi
echo "Check the workflow logs for details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_STEP_SUMMARY echo "Check the workflow logs for details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_STEP_SUMMARY
.github/workflows/build-iperf3.yml
+91 -91
View File
@@ -2,6 +2,12 @@ name: Build iperf3 Static Binaries
on: on:
workflow_dispatch: workflow_dispatch:
schedule:
# check upstream for a new iperf3 release once a day
- cron: '47 5 * * *'
permissions:
contents: write
jobs: jobs:
check-release: check-release:
@@ -10,22 +16,23 @@ jobs:
latest-version: ${{ steps.get-version.outputs.version }} latest-version: ${{ steps.get-version.outputs.version }}
should-build: ${{ steps.check-version.outputs.should-build }} should-build: ${{ steps.check-version.outputs.should-build }}
steps: steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Get latest iperf3 release - name: Get latest iperf3 release
id: get-version id: get-version
run: | run: |
# Get the latest release tag from GitHub API # Get the latest release tag from GitHub API (e.g. "3.19")
LATEST_VERSION=$(curl -s https://api.github.com/repos/esnet/iperf/releases/latest | jq -r '.tag_name') LATEST_VERSION=$(curl -4 --retry 5 --retry-delay 2 --connect-timeout 15 -s https://api.github.com/repos/esnet/iperf/releases/latest | jq -r '.tag_name')
echo "Latest iperf3 version: $LATEST_VERSION" echo "Latest iperf3 version: $LATEST_VERSION"
if [ -z "$LATEST_VERSION" ] || [ "$LATEST_VERSION" = "null" ]; then
echo "Failed to determine latest iperf3 version"
exit 1
fi
echo "version=$LATEST_VERSION" >> $GITHUB_OUTPUT echo "version=$LATEST_VERSION" >> $GITHUB_OUTPUT
- name: Check if version exists in releases - name: Check if version exists in releases
id: check-version id: check-version
run: | run: |
VERSION="${{ steps.get-version.outputs.version }}" VERSION="${{ steps.get-version.outputs.version }}"
# Check if this version already exists in our releases # Check if this version already exists in our releases
if gh release view "iperf3-$VERSION" --repo ${{ github.repository }} >/dev/null 2>&1; then if gh release view "iperf3-$VERSION" --repo ${{ github.repository }} >/dev/null 2>&1; then
echo "Version $VERSION already exists in releases" echo "Version $VERSION already exists in releases"
@@ -41,7 +48,11 @@ jobs:
needs: check-release needs: check-release
if: needs.check-release.outputs.should-build == 'true' if: needs.check-release.outputs.should-build == 'true'
runs-on: ubuntu-latest runs-on: ubuntu-latest
# allow the ARM 32-bit leg to fail without failing the workflow (iperf3 arm32
# builds are historically flaky; kept as a safety net for future versions)
continue-on-error: ${{ matrix.arch == 'arm' }}
strategy: strategy:
fail-fast: false
matrix: matrix:
include: include:
- arch: x64 - arch: x64
@@ -53,81 +64,66 @@ jobs:
- arch: aarch64 - arch: aarch64
cross: aarch64-linux-musl cross: aarch64-linux-musl
host: aarch64-linux-gnu host: aarch64-linux-gnu
# Note: ARM 32-bit compilation of iperf3 >3.15 has known issues
# We'll still attempt it but expect potential failures
- arch: arm - arch: arm
cross: arm-linux-musleabihf cross: arm-linux-musleabihf
host: arm-linux-gnueabihf host: arm-linux-gnueabihf
steps: steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Create compilation script - name: Create compilation script
run: | run: |
VERSION="${{ needs.check-release.outputs.latest-version }}"
ARCH="${{ matrix.arch }}"
CROSS="${{ matrix.cross }}"
HOST="${{ matrix.host }}"
# Single script for all architectures using musl cross-compilation # Single script for all architectures using musl cross-compilation
cat > compile-iperf3.sh << EOF cat > compile-iperf3.sh << 'EOF'
#!/bin/bash #!/bin/bash
set -e set -e
# Activate Holy Build Box lib compilation environment # Activate Holy Build Box lib compilation environment
source /hbb/activate source /hbb/activate
set -x set -x
# remove obsolete CentOS repos # remove obsolete CentOS repos
cd /etc/yum.repos.d/ cd /etc/yum.repos.d/
rm -f CentOS-Base.repo CentOS-SCLo-scl-rh.repo CentOS-SCLo-scl.repo CentOS-fasttrack.repo CentOS-x86_64-kernel.repo rm -f CentOS-Base.repo CentOS-SCLo-scl-rh.repo CentOS-SCLo-scl.repo CentOS-fasttrack.repo CentOS-x86_64-kernel.repo
yum install -y yum-plugin-ovl # fix for docker overlay fs yum install -y yum-plugin-ovl # fix for docker overlay fs
yum install -y xz yum install -y xz
# download musl cross compilation toolchain # download musl cross compilation toolchain
cd ~ cd ~
curl -L "https://musl.cc/\$CROSS-cross.tgz" -o "\$CROSS-cross.tgz" curl -L -4 --retry 5 --retry-delay 2 --connect-timeout 15 "https://musl.cc/$CROSS-cross.tgz" -o "$CROSS-cross.tgz"
tar xf "\$CROSS-cross.tgz" tar xf "$CROSS-cross.tgz"
# Activate Holy Build Box exe compilation environment # Activate Holy Build Box exe compilation environment
source /hbb_exe/activate source /hbb_exe/activate
# download and compile iperf3 # download and compile iperf3
cd ~ cd ~
curl -L "https://github.com/esnet/iperf/archive/\$VERSION.tar.gz" -o "iperf.tar.gz" curl -L -4 --retry 5 --retry-delay 2 --connect-timeout 15 "https://github.com/esnet/iperf/archive/$VERSION.tar.gz" -o "iperf.tar.gz"
tar xf iperf.tar.gz tar xf iperf.tar.gz
cd iperf-* cd iperf-*
CC=/root/\$CROSS-cross/bin/\$CROSS-gcc ./configure --disable-shared --disable-profiling --build x86_64-pc-linux-gnu --host "\$HOST" --with-openssl=no --enable-static-bin CC=/root/$CROSS-cross/bin/$CROSS-gcc ./configure --disable-shared --disable-profiling --build x86_64-pc-linux-gnu --host "$HOST" --with-openssl=no --enable-static-bin
# remove libatomic.la so libtool links the static libatomic.a rather than
# attempting to link libatomic.so into the static binary (breaks arm32 builds)
rm -f /root/$CROSS-cross/$CROSS/lib/libatomic.la
make make
# verify no external shared library links # verify no external shared library links
libcheck src/iperf3 libcheck src/iperf3
# copy iperf3 binary to mounted dir # copy iperf3 binary to mounted dir
cp src/iperf3 "/io/iperf3_\$ARCH" cp src/iperf3 "/io/iperf3_$ARCH"
EOF EOF
chmod +x compile-iperf3.sh chmod +x compile-iperf3.sh
- name: Compile iperf3 binary - name: Compile iperf3 binary
run: | run: |
ARCH="${{ matrix.arch }}"
CROSS="${{ matrix.cross }}"
HOST="${{ matrix.host }}"
# Use musl cross-compilation for all architectures # Use musl cross-compilation for all architectures
docker run --rm -v $(pwd):/io --env ARCH=$ARCH --env CROSS=$CROSS --env HOST=$HOST --env VERSION="${{ needs.check-release.outputs.latest-version }}" phusion/holy-build-box-64:latest bash /io/compile-iperf3.sh docker run --rm -v $(pwd):/io --env ARCH=${{ matrix.arch }} --env CROSS=${{ matrix.cross }} --env HOST=${{ matrix.host }} --env VERSION="${{ needs.check-release.outputs.latest-version }}" phusion/holy-build-box-64:latest bash /io/compile-iperf3.sh
- name: Verify binary - name: Verify binary
run: | run: |
ARCH="${{ matrix.arch }}" ls -la iperf3_${{ matrix.arch }}
ls -la iperf3_$ARCH file iperf3_${{ matrix.arch }}
file iperf3_$ARCH
- name: Upload binary as artifact - name: Upload binary as artifact
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
@@ -135,8 +131,6 @@ jobs:
name: iperf3_${{ matrix.arch }} name: iperf3_${{ matrix.arch }}
path: iperf3_${{ matrix.arch }} path: iperf3_${{ matrix.arch }}
retention-days: 1 retention-days: 1
# Allow arm builds to fail as ARM 32-bit has known compilation issues with newer versions
continue-on-error: ${{ matrix.arch == 'arm' }}
virustotal-scan: virustotal-scan:
needs: [check-release, build] needs: [check-release, build]
@@ -157,7 +151,7 @@ jobs:
mkdir -p scan-binaries mkdir -p scan-binaries
find artifacts -name "iperf3_*" -type f -exec cp {} scan-binaries/ \; || true find artifacts -name "iperf3_*" -type f -exec cp {} scan-binaries/ \; || true
ls -la scan-binaries/ ls -la scan-binaries/
# Check if we have at least the x64 binary # Check if we have at least the x64 binary
if [ ! -f "scan-binaries/iperf3_x64" ]; then if [ ! -f "scan-binaries/iperf3_x64" ]; then
echo "Error: No iperf3_x64 binary found" echo "Error: No iperf3_x64 binary found"
@@ -170,38 +164,35 @@ jobs:
echo "## VirusTotal Scan Results" > scan_results.md echo "## VirusTotal Scan Results" > scan_results.md
echo "| Binary | Status | Malicious | Suspicious | Undetected | VirusTotal URL |" >> scan_results.md echo "| Binary | Status | Malicious | Suspicious | Undetected | VirusTotal URL |" >> scan_results.md
echo "|--------|--------|-----------|------------|------------|----------------|" >> scan_results.md echo "|--------|--------|-----------|------------|------------|----------------|" >> scan_results.md
ALL_CLEAN=true ALL_CLEAN=true
SCAN_DATA=""
for binary in scan-binaries/iperf3_*; do for binary in scan-binaries/iperf3_*; do
filename=$(basename "$binary") filename=$(basename "$binary")
echo "Uploading $filename to VirusTotal..." echo "Uploading $filename to VirusTotal..."
# Upload to VirusTotal # Upload to VirusTotal
upload_response=$(curl -s --request POST \ upload_response=$(curl -4 --retry 5 --retry-delay 2 --connect-timeout 15 -s --request POST \
--url https://www.virustotal.com/api/v3/files \ --url https://www.virustotal.com/api/v3/files \
--header 'accept: application/json' \ --header 'accept: application/json' \
--header 'content-type: multipart/form-data' \ --header 'content-type: multipart/form-data' \
--header "x-apikey: ${{ secrets.VIRUSTOTAL_API_KEY }}" \ --header "x-apikey: ${{ secrets.VIRUSTOTAL_API_KEY }}" \
--form "file=@$binary") --form "file=@$binary")
analysis_id=$(echo "$upload_response" | jq -r '.data.id') analysis_id=$(echo "$upload_response" | jq -r '.data.id')
echo "Analysis ID for $filename: $analysis_id" echo "Analysis ID for $filename: $analysis_id"
if [ "$analysis_id" = "null" ] || [ -z "$analysis_id" ]; then if [ "$analysis_id" = "null" ] || [ -z "$analysis_id" ]; then
echo "Failed to upload $filename to VirusTotal" echo "Failed to upload $filename to VirusTotal"
echo "| $filename | Upload Failed | N/A | N/A | N/A | N/A |" >> scan_results.md echo "| $filename | Upload Failed | N/A | N/A | N/A | N/A |" >> scan_results.md
ALL_CLEAN=false ALL_CLEAN=false
continue continue
fi fi
# Store analysis ID for later retrieval # Store analysis ID for later retrieval
echo "${filename}:${analysis_id}" >> analysis_ids.txt echo "${filename}:${analysis_id}" >> analysis_ids.txt
done done
echo "all_clean_upload=$ALL_CLEAN" >> $GITHUB_OUTPUT
# Wait 2 minutes for scans to complete # Wait 2 minutes for scans to complete
echo "Waiting 2 minutes for VirusTotal scans to complete..." echo "Waiting 2 minutes for VirusTotal scans to complete..."
sleep 120 sleep 120
@@ -210,60 +201,61 @@ jobs:
id: get-results id: get-results
run: | run: |
ALL_CLEAN=true ALL_CLEAN=true
SCAN_SUMMARY=""
while IFS=':' read -r filename analysis_id; do while IFS=':' read -r filename analysis_id; do
echo "Retrieving results for $filename (ID: $analysis_id)..." echo "Retrieving results for $filename (ID: $analysis_id)..."
# Get scan results # Get scan results
result_response=$(curl -s --request GET \ result_response=$(curl -4 --retry 5 --retry-delay 2 --connect-timeout 15 -s --request GET \
--url "https://www.virustotal.com/api/v3/analyses/$analysis_id" \ --url "https://www.virustotal.com/api/v3/analyses/$analysis_id" \
--header 'accept: application/json' \ --header 'accept: application/json' \
--header "x-apikey: ${{ secrets.VIRUSTOTAL_API_KEY }}") --header "x-apikey: ${{ secrets.VIRUSTOTAL_API_KEY }}")
status=$(echo "$result_response" | jq -r '.data.attributes.status // "unknown"') status=$(echo "$result_response" | jq -r '.data.attributes.status // "unknown"')
if [ "$status" = "completed" ]; then if [ "$status" = "completed" ]; then
malicious=$(echo "$result_response" | jq -r '.data.attributes.stats.malicious // 0') malicious=$(echo "$result_response" | jq -r '.data.attributes.stats.malicious // 0')
suspicious=$(echo "$result_response" | jq -r '.data.attributes.stats.suspicious // 0') suspicious=$(echo "$result_response" | jq -r '.data.attributes.stats.suspicious // 0')
undetected=$(echo "$result_response" | jq -r '.data.attributes.stats.undetected // 0') undetected=$(echo "$result_response" | jq -r '.data.attributes.stats.undetected // 0')
# Get file hash from the item link # Get file hash from the item link
item_link=$(echo "$result_response" | jq -r '.data.links.item // ""') item_link=$(echo "$result_response" | jq -r '.data.links.item // ""')
file_hash=$(echo "$item_link" | sed 's/.*files\///') file_hash=$(echo "$item_link" | sed 's/.*files\///')
vt_url="https://www.virustotal.com/gui/file/$file_hash" vt_url="https://www.virustotal.com/gui/file/$file_hash"
if [ "$malicious" -gt 0 ] || [ "$suspicious" -gt 0 ]; then if [ "$malicious" -gt 0 ] || [ "$suspicious" -gt 0 ]; then
status_text="⚠️ FLAGGED" status_text="⚠️ FLAGGED"
ALL_CLEAN=false ALL_CLEAN=false
else else
status_text="✅ Clean" status_text="✅ Clean"
fi fi
echo "| $filename | $status_text | $malicious | $suspicious | $undetected | [$file_hash]($vt_url) |" >> scan_results.md echo "| $filename | $status_text | $malicious | $suspicious | $undetected | [$file_hash]($vt_url) |" >> scan_results.md
else else
echo "| $filename | ⏳ Pending | N/A | N/A | N/A | Scan not completed |" >> scan_results.md echo "| $filename | ⏳ Pending | N/A | N/A | N/A | Scan not completed |" >> scan_results.md
ALL_CLEAN=false ALL_CLEAN=false
fi fi
done < analysis_ids.txt done < analysis_ids.txt
echo "all_clean=$ALL_CLEAN" >> $GITHUB_OUTPUT echo "all_clean=$ALL_CLEAN" >> $GITHUB_OUTPUT
# Add summary to GitHub step summary # Add summary to GitHub step summary
cat scan_results.md >> $GITHUB_STEP_SUMMARY cat scan_results.md >> $GITHUB_STEP_SUMMARY
- name: Create scan summary - name: Create scan summary
id: scan-summary id: scan-summary
run: | run: |
RESULTS=$(cat scan_results.md)
ALL_CLEAN="${{ steps.get-results.outputs.all_clean }}" ALL_CLEAN="${{ steps.get-results.outputs.all_clean }}"
# Escape newlines for GitHub output # Pass the scan results through as a multi-line output (heredoc delimiter
RESULTS_ESCAPED=$(echo "$RESULTS" | sed ':a;N;$!ba;s/\n/\\n/g') # syntax) so the markdown table renders correctly in the release body
{
echo "results=$RESULTS_ESCAPED" >> $GITHUB_OUTPUT echo "results<<RESULTS_EOF"
cat scan_results.md
echo "RESULTS_EOF"
} >> $GITHUB_OUTPUT
echo "all-clean=$ALL_CLEAN" >> $GITHUB_OUTPUT echo "all-clean=$ALL_CLEAN" >> $GITHUB_OUTPUT
- name: Upload scan results as artifact - name: Upload scan results as artifact
@@ -280,9 +272,6 @@ jobs:
if: needs.check-release.outputs.should-build == 'true' && needs.virustotal-scan.outputs.all-clean == 'true' if: needs.check-release.outputs.should-build == 'true' && needs.virustotal-scan.outputs.all-clean == 'true'
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Download all artifacts - name: Download all artifacts
uses: actions/download-artifact@v4 uses: actions/download-artifact@v4
with: with:
@@ -296,13 +285,20 @@ jobs:
# Move all binaries to release-assets directory # Move all binaries to release-assets directory
find artifacts -name "iperf3_*" -type f -exec cp {} release-assets/ \; || true find artifacts -name "iperf3_*" -type f -exec cp {} release-assets/ \; || true
ls -la release-assets/ ls -la release-assets/
# Check if we have at least the x64 binary # Check if we have at least the x64 binary
if [ ! -f "release-assets/iperf3_x64" ]; then if [ ! -f "release-assets/iperf3_x64" ]; then
echo "Error: No iperf3_x64 binary found" echo "Error: No iperf3_x64 binary found"
exit 1 exit 1
fi fi
- name: Download upstream source tarball
run: |
# Attach the exact upstream source these binaries were built from
# (iperf3 is BSD-3-Clause; retain the copyright notice with distribution)
curl -L --retry 5 --retry-delay 2 --connect-timeout 15 "https://github.com/esnet/iperf/archive/${{ needs.check-release.outputs.latest-version }}.tar.gz" -o "release-assets/iperf-${{ needs.check-release.outputs.latest-version }}.tar.gz"
ls -la release-assets/
- name: Generate SHA256 checksums - name: Generate SHA256 checksums
run: | run: |
cd release-assets cd release-assets
@@ -330,38 +326,42 @@ jobs:
path: vt-results path: vt-results
- name: Create Release - name: Create Release
uses: softprops/action-gh-release@v1 uses: softprops/action-gh-release@v2
with: with:
tag_name: iperf3-${{ needs.check-release.outputs.latest-version }} tag_name: iperf3-${{ needs.check-release.outputs.latest-version }}
name: iperf3 ${{ needs.check-release.outputs.latest-version }} Static Binaries name: iperf3 ${{ needs.check-release.outputs.latest-version }} Static Binaries
body: | body: |
Static binaries for iperf3 ${{ needs.check-release.outputs.latest-version }} Static binaries for iperf3 ${{ needs.check-release.outputs.latest-version }}
Built using musl toolchains for maximum compatibility. Built using musl toolchains for maximum compatibility.
**Architectures:** **Architectures:**
- `iperf3_x64` - x86_64 (64-bit) - `iperf3_x64` - x86_64 (64-bit)
- `iperf3_x86` - i686 (32-bit) - `iperf3_x86` - i686 (32-bit)
- `iperf3_aarch64` - ARM 64-bit - `iperf3_aarch64` - ARM 64-bit
- `iperf3_arm` - ARM 32-bit (if available) - `iperf3_arm` - ARM 32-bit (if available)
${{ steps.check-arm.outputs.arm-note }} ${{ steps.check-arm.outputs.arm-note }}
**Security Verification:** **Security Verification:**
All binaries have been scanned by VirusTotal and verified clean. All binaries have been scanned by VirusTotal and verified clean.
${{ needs.virustotal-scan.outputs.scan-results }} ${{ needs.virustotal-scan.outputs.scan-results }}
**Checksum Verification:** **Checksum Verification:**
```bash ```bash
# Verify checksums # Verify checksums
sha256sum -c iperf3-checksums.sha256 sha256sum -c iperf3-checksums.sha256
``` ```
For usage in YABS script, place these binaries in the `bin/iperf/` directory. **License & Source:**
iperf3 is licensed under [BSD-3-Clause](https://github.com/esnet/iperf/blob/master/LICENSE) (© ESnet / LBL).
These are unofficial static builds produced by CI for use by [YABS](https://github.com/masonr/yet-another-bench-script);
the exact upstream source is attached as `iperf-${{ needs.check-release.outputs.latest-version }}.tar.gz`.
files: | files: |
release-assets/iperf3_* release-assets/iperf3_*
release-assets/iperf3-checksums.sha256 release-assets/iperf3-checksums.sha256
release-assets/*.tar.gz
draft: false draft: false
prerelease: false prerelease: false
env: env:
@@ -376,7 +376,7 @@ jobs:
run: | run: |
echo "## ⚠️ iperf3 Build Failed" >> $GITHUB_STEP_SUMMARY echo "## ⚠️ iperf3 Build Failed" >> $GITHUB_STEP_SUMMARY
echo "Version: ${{ needs.check-release.outputs.latest-version }}" >> $GITHUB_STEP_SUMMARY echo "Version: ${{ needs.check-release.outputs.latest-version }}" >> $GITHUB_STEP_SUMMARY
if [ "${{ needs.virustotal-scan.outputs.all-clean }}" == "false" ]; then if [ "${{ needs.virustotal-scan.outputs.all-clean }}" == "false" ]; then
echo "**Reason:** VirusTotal scan detected issues with one or more binaries" >> $GITHUB_STEP_SUMMARY echo "**Reason:** VirusTotal scan detected issues with one or more binaries" >> $GITHUB_STEP_SUMMARY
echo "**Scan Results:**" >> $GITHUB_STEP_SUMMARY echo "**Scan Results:**" >> $GITHUB_STEP_SUMMARY
@@ -384,5 +384,5 @@ jobs:
else else
echo "**Reason:** Build compilation failed" >> $GITHUB_STEP_SUMMARY echo "**Reason:** Build compilation failed" >> $GITHUB_STEP_SUMMARY
fi fi
echo "Check the workflow logs for details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_STEP_SUMMARY echo "Check the workflow logs for details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_STEP_SUMMARY
.github/workflows/monitor-releases.yml
-74
View File
@@ -1,74 +0,0 @@
name: Monitor External Releases for Updates
on:
schedule:
# Check every hour for new releases
- cron: '0 * * * *'
workflow_dispatch:
jobs:
monitor-releases:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Check fio releases
id: check-fio
run: |
# Get latest fio release
LATEST_FIO=$(curl -s https://api.github.com/repos/axboe/fio/releases/latest | jq -r '.tag_name')
echo "Latest fio: $LATEST_FIO"
# Check if we already have this version
if ! gh release view "fio-$LATEST_FIO" --repo ${{ github.repository }} >/dev/null 2>&1; then
echo "New fio version found: $LATEST_FIO"
echo "trigger-fio=true" >> $GITHUB_OUTPUT
echo "fio-version=$LATEST_FIO" >> $GITHUB_OUTPUT
else
echo "fio $LATEST_FIO already exists"
echo "trigger-fio=false" >> $GITHUB_OUTPUT
fi
env:
GH_TOKEN: ${{ github.token }}
- name: Check iperf3 releases
id: check-iperf3
run: |
# Get latest iperf3 release
LATEST_IPERF3=$(curl -s https://api.github.com/repos/esnet/iperf/releases/latest | jq -r '.tag_name')
echo "Latest iperf3: $LATEST_IPERF3"
# Check if we already have this version
if ! gh release view "iperf3-$LATEST_IPERF3" --repo ${{ github.repository }} >/dev/null 2>&1; then
echo "New iperf3 version found: $LATEST_IPERF3"
echo "trigger-iperf3=true" >> $GITHUB_OUTPUT
echo "iperf3-version=$LATEST_IPERF3" >> $GITHUB_OUTPUT
else
echo "iperf3 $LATEST_IPERF3 already exists"
echo "trigger-iperf3=false" >> $GITHUB_OUTPUT
fi
env:
GH_TOKEN: ${{ github.token }}
- name: Trigger fio build workflow
if: steps.check-fio.outputs.trigger-fio == 'true'
run: |
echo "Triggering fio build for version ${{ steps.check-fio.outputs.fio-version }}"
gh workflow run build-fio.yml --repo ${{ github.repository }}
env:
GH_TOKEN: ${{ github.token }}
- name: Trigger iperf3 build workflow
if: steps.check-iperf3.outputs.trigger-iperf3 == 'true'
run: |
echo "Triggering iperf3 build for version ${{ steps.check-iperf3.outputs.iperf3-version }}"
gh workflow run build-iperf3.yml --repo ${{ github.repository }}
env:
GH_TOKEN: ${{ github.token }}
- name: Summary
run: |
echo "## Release Monitoring Summary" >> $GITHUB_STEP_SUMMARY
echo "- fio: ${{ steps.check-fio.outputs.trigger-fio == 'true' && 'New version triggered build' || 'No new version' }}" >> $GITHUB_STEP_SUMMARY
echo "- iperf3: ${{ steps.check-iperf3.outputs.trigger-iperf3 == 'true' && 'New version triggered build' || 'No new version' }}" >> $GITHUB_STEP_SUMMARY