1# This is a workflow triggered by PR or triggered manually
2# Runs quick performance tests and reports the comparison against HEAD
3# Test should take less than 10 minutes to run on current self-hosted devices
4name: "Performance Testing"
5
6# Controls when the action will run.
7# This workflow runs when manually triggered by keywords used in the start of a review comment
8# Currently that phrase is /bench_x64. /bench_aarch64 and /bench_all are TODOs.
9on:
10  issue_comment:
11    types: [created]
12  push:
13
14# Env variables
15env:
16  SG_COMMIT: 2ab01ac
17  GITHUB_CONTEXT: ${{ toJson(github) }}
18
19jobs:
20  Wasmtime_Repo_On_PR_Comment:
21    name: Benchmark x64 on PR comment Wasmtime repo
22    runs-on: ubuntu-latest
23    if: |
24      (github.event_name == 'issue_comment') &&
25      (github.event.issue.pull_request.url) &&
26      (contains(github.event.comment.body, '/bench_x64')) &&
27      (('abrown' == github.event.comment.user.login)
28        || ('afonso360' == github.event.comment.user.login)
29        || ('akirilov-arm' == github.event.comment.user.login)
30        || ('alexcrichton' == github.event.comment.user.login)
31        || ('bbouvier' == github.event.comment.user.login)
32        || ('bjorn3' == github.event.comment.user.login)
33        || ('cfallin' == github.event.comment.user.login)
34        || ('elliottt' == github.event.comment.user.login)
35        || ('fitzgen' == github.event.comment.user.login)
36        || ('jameysharp' == github.event.comment.user.login)
37        || ('jlb6740' == github.event.comment.user.login)
38        || ('sparker-arm' == github.event.comment.user.login)
39        || ('uweigand' == github.event.comment.user.login))
40    steps:
41      - run: echo "$GITHUB_CONTEXT"
42      - run: |
43          # Create and Push Branch
44          git clone https://wasmtime-publish:${{secrets.PERSONAL_ACCESS_TOKEN}}@github.com/bytecodealliance/wasmtime-sightglass-benchmarking.git
45          cd wasmtime-sightglass-benchmarking
46          git remote add wasmtime ${{ github.event.repository.clone_url }}
47          git fetch wasmtime refs/pull/*/merge:refs/remotes/wasmtime/pull/*/merge
48          export issue_pr_url=${{ github.event.issue.pull_request.url }}
49          export issue_commits_url=${{ github.event.issue.comments_url }}
50          export issue_ref_name=$(curl -sSL $issue_pr_url | jq -r '.head.ref' | head -n 1)
51          export issue_number=$(curl -sSL $issue_pr_url | jq -r '.number' | head -n 1)
52          export issue_merge_commit_sha=$(curl -sSL $issue_pr_url | jq -r '.merge_commit_sha' | head -n 1)
53          git submodule update --init --recursive
54          git checkout wasmtime/pull/${issue_number}/merge -b pull/${issue_number}/merge/${issue_merge_commit_sha}
55          git config user.name $(curl -sSL $issue_commits_url | jq -r '.[].commit.committer.name' | tail -n 1)
56          git config user.email $(curl -sSL $issue_commits_url | jq -r '.[].commit.committer.email' | tail -n 1)
57          git log -n 1
58          git commit --allow-empty -m "${issue_commits_url}"
59          git push origin --force pull/${issue_number}/merge/${issue_merge_commit_sha}
60          git log -n 1
61
62  Performance_Repo_On_Push:
63    name: Benchmark x64 on push Performance repo
64    runs-on: [self-hosted, linux, x64]
65    if: (github.event_name == 'push') && (github.repository == 'bytecodealliance/wasmtime-sightglass-benchmarking')
66    steps:
67      - run: echo "$GITHUB_CONTEXT"
68      - run: echo "${{ github.event.head_commit.message }}"
69      - name: "Build sightglass commit '${{ env.SG_COMMIT }}'"
70        run: |
71          cd ../ && ls -l && rm -rf ./sightglass
72          git clone https://github.com/bytecodealliance/sightglass.git && cd ./sightglass
73          git checkout ${{env.SG_COMMIT}}
74          cargo build --release
75
76      - name: Checkout patch from bytecodealliance/wasmtime (pushed and triggering on this perf repo)
77        uses: actions/checkout@v6
78        with:
79          submodules: true
80          path: wasmtime_commit
81
82      - run: rustup update nightly && rustup default nightly
83
84      - name: Build patch from bytecodealliance/wasmtime (pushed and triggering on this perf repo)
85        working-directory: ./wasmtime_commit
86        run: |
87          cargo --version
88          cargo build --release -p wasmtime-bench-api
89          cp target/release/libwasmtime_bench_api.so /tmp/wasmtime_commit.so
90
91      - name: Checkout main from bytecodealliance/wasmtime
92        uses: actions/checkout@v6
93        with:
94          ref: 'main'
95          repository: 'bytecodealliance/wasmtime'
96          submodules: true
97          path: wasmtime_main
98
99      - name: Build main from bytecodealliance/wasmtime
100        working-directory: ./wasmtime_main
101        run: |
102          cargo build --release -p wasmtime-bench-api
103          cp target/release/libwasmtime_bench_api.so /tmp/wasmtime_main.so
104
105      - name: Run performance tests
106        working-directory: ../sightglass
107        run: |
108          cargo run -- \
109          benchmark \
110          --processes 5 \
111          --iterations-per-process 5 \
112          --engine /tmp/wasmtime_main.so \
113          --engine /tmp/wasmtime_commit.so \
114          --output-file /tmp/results.txt
115
116      - name: Print Results
117        run: cat /tmp/results.txt
118
119      - id: get-comment-body
120        name: Create Results Body
121        run: |
122            body="$(cat /tmp/results.txt)"
123            body="${body//'%'/'%25'}"
124            body="${body//$'\n'/'%0A'}"
125            body="${body//$'\r'/'%0D'}"
126            echo "::set-output name=body::$body"
127
128      - name: Publish Results
129        run: |
130          curl -X POST -H "Accept: application/vnd.github.v3+json" \
131          -H "Authorization: token ${{ secrets.WASMTIME_PUBLISHING_TOKEN }}" \
132          ${{ github.event.head_commit.message }} \
133          -d '{"body": ${{ toJSON(steps.get-comment-body.outputs.body) }}}'
134