1# This is a workflow triggered by PR or triggered manually
2# Runs quick performance tests and reports the comparison against HEAD
3# Test should take less than 10 minutes to run on current self-hosted devices
4name: "Performance Testing"
5
6# Controls when the action will run.
7# This workflow runs when manually triggered by keywords used in the start of a review comment
8# Currently that phrase is /bench_x64. /bench_aarch64 and /bench_all are TODOs.
9on:
10  issue_comment:
11    types: [created]
12  push:
13
14# Env variables
15env:
16  SG_COMMIT: 2ab01ac
17  GITHUB_CONTEXT: ${{ toJson(github) }}
18
19jobs:
20  Wasmtime_Repo_On_PR_Comment:
21    name: Benchmark x64 on PR comment Wasmtime repo
22    runs-on: ubuntu-latest
23    if: |
24      (github.event_name == 'issue_comment') &&
25      (github.event.issue.pull_request.url) &&
26      (contains(github.event.comment.body, '/bench_x64')) &&
27      (('abrown' == github.event.comment.user.login)
28        || ('afonso360' == github.event.comment.user.login)
29        || ('akirilov-arm' == github.event.comment.user.login)
30        || ('alexcrichton' == github.event.comment.user.login)
31        || ('bbouvier' == github.event.comment.user.login)
32        || ('bjorn3' == github.event.comment.user.login)
33        || ('cfallin' == github.event.comment.user.login)
34        || ('fitzgen' == github.event.comment.user.login)
35        || ('jlb6740' == github.event.comment.user.login)
36        || ('sparker-arm' == github.event.comment.user.login)
37        || ('uweigand' == github.event.comment.user.login))
38    steps:
39      - run: echo "$GITHUB_CONTEXT"
40      - run: |
41          # Create and Push Branch
42          git clone https://wasmtime-publish:${{secrets.PERSONAL_ACCESS_TOKEN}}@github.com/bytecodealliance/wasmtime-sightglass-benchmarking.git
43          cd wasmtime-sightglass-benchmarking
44          git remote add wasmtime ${{ github.event.repository.clone_url }}
45          git fetch wasmtime refs/pull/*/merge:refs/remotes/wasmtime/pull/*/merge
46          export issue_pr_url=${{ github.event.issue.pull_request.url }}
47          export issue_commits_url=${{ github.event.issue.comments_url }}
48          export issue_ref_name=$(curl -sSL $issue_pr_url | jq -r '.head.ref' | head -n 1)
49          export issue_number=$(curl -sSL $issue_pr_url | jq -r '.number' | head -n 1)
50          export issue_merge_commit_sha=$(curl -sSL $issue_pr_url | jq -r '.merge_commit_sha' | head -n 1)
51          git submodule update --init --recursive
52          git checkout wasmtime/pull/${issue_number}/merge -b pull/${issue_number}/merge/${issue_merge_commit_sha}
53          git config user.name $(curl -sSL $issue_commits_url | jq -r '.[].commit.committer.name' | tail -n 1)
54          git config user.email $(curl -sSL $issue_commits_url | jq -r '.[].commit.committer.email' | tail -n 1)
55          git log -n 1
56          git commit --allow-empty -m "${issue_commits_url}"
57          git push origin --force pull/${issue_number}/merge/${issue_merge_commit_sha}
58          git log -n 1
59
60  Performance_Repo_On_Push:
61    name: Benchmark x64 on push Performance repo
62    runs-on: [self-hosted, linux, x64]
63    if: (github.event_name == 'push') && (github.repository == 'bytecodealliance/wasmtime-sightglass-benchmarking')
64    steps:
65      - run: echo "$GITHUB_CONTEXT"
66      - run: echo "${{ github.event.head_commit.message }}"
67      - name: "Build sightglass commit '${{ env.SG_COMMIT }}'"
68        run: |
69          cd ../ && ls -l && rm -rf ./sightglass
70          git clone https://github.com/bytecodealliance/sightglass.git && cd ./sightglass
71          git checkout ${{env.SG_COMMIT}}
72          cargo build --release
73
74      - name: Checkout patch from bytecodealliance/wasmtime (pushed and triggering on this perf repo)
75        uses: actions/checkout@v3
76        with:
77          submodules: true
78          path: wasmtime_commit
79
80      - run: rustup update nightly && rustup default nightly
81
82      - name: Build patch from bytecodealliance/wasmtime (pushed and triggering on this perf repo)
83        working-directory: ./wasmtime_commit
84        run: |
85          cargo --version
86          cargo build --release -p wasmtime-bench-api
87          cp target/release/libwasmtime_bench_api.so /tmp/wasmtime_commit.so
88
89      - name: Checkout main from bytecodealliance/wasmtime
90        uses: actions/checkout@v3
91        with:
92          ref: 'main'
93          repository: 'bytecodealliance/wasmtime'
94          submodules: true
95          path: wasmtime_main
96
97      - name: Build main from bytecodealliance/wasmtime
98        working-directory: ./wasmtime_main
99        run: |
100          cargo build --release -p wasmtime-bench-api
101          cp target/release/libwasmtime_bench_api.so /tmp/wasmtime_main.so
102
103      - name: Run performance tests
104        working-directory: ../sightglass
105        run: |
106          cargo run -- \
107          benchmark \
108          --processes 5 \
109          --iterations-per-process 5 \
110          --engine /tmp/wasmtime_main.so \
111          --engine /tmp/wasmtime_commit.so \
112          --output-file /tmp/results.txt
113
114      - name: Print Results
115        run: cat /tmp/results.txt
116
117      - id: get-comment-body
118        name: Create Results Body
119        run: |
120            body="$(cat /tmp/results.txt)"
121            body="${body//'%'/'%25'}"
122            body="${body//$'\n'/'%0A'}"
123            body="${body//$'\r'/'%0D'}"
124            echo "::set-output name=body::$body"
125
126      - name: Publish Results
127        run: |
128          curl -X POST -H "Accept: application/vnd.github.v3+json" \
129          -H "Authorization: token ${{ secrets.WASMTIME_PUBLISHING_TOKEN }}" \
130          ${{ github.event.head_commit.message }} \
131          -d '{"body": ${{ toJSON(steps.get-comment-body.outputs.body) }}}'
132