1# This is a workflow triggered by PR or triggered manually
2# Runs quick performance tests and reports the comparison against HEAD
3# Test should take less than 10 minutes to run on current self-hosted devices
4name: "Performance Testing"
5
6# Controls when the action will run.
7# This workflow runs when manually triggered by keywords used in the start of a review comment
8# Currently that phrase is /bench_x64. /bench_aarch64 and /bench_all are TODOs.
9on:
10  pull_request_review:
11    types: [submitted, edited]
12  push:
13
14# Env variables
15env:
16  SG_COMMIT: b4971ae
17  GITHUB_CONTEXT: ${{ toJson(github) }}
18
19jobs:
20  Wasmtime_Repo_On_PR_Comment:
21    name: Benchmark x64 on PR comment Wasmtime repo
22    runs-on: ubuntu-latest
23    if: |
24      (github.event_name == 'pull_request_review') &&
25      (contains(github.event.review.body, '/bench_x64')) &&
26      (('abrown' == github.event.review.user.login)
27        || ('afonso360' == github.event.review.user.login)
28        || ('akirilov-arm' == github.event.review.user.login)
29        || ('alexcrichton' == github.event.review.user.login)
30        || ('bbouvier' == github.event.review.user.login)
31        || ('bjorn3' == github.event.review.user.login)
32        || ('cfallin' == github.event.review.user.login)
33        || ('fitzgen' == github.event.review.user.login)
34        || ('jlb6740' == github.event.review.user.login)
35        || ('sparker-arm' == github.event.review.user.login)
36        || ('uweigand' == github.event.review.user.login))
37    steps:
38      - run: echo "$GITHUB_CONTEXT"
39      - run: |
40          # Create and Push Branch
41          git clone https://wasmtime-publish:${{secrets.PERSONAL_ACCESS_TOKEN}}@github.com/bytecodealliance/wasmtime-sightglass-benchmarking.git
42          cd wasmtime-sightglass-benchmarking
43          git remote add wasmtime ${{ github.event.repository.clone_url }}
44          git fetch wasmtime refs/pull/*/merge:refs/remotes/wasmtime/pull/*/merge
45          git checkout wasmtime/pull/${{ github.ref_name }} -b ${{ github.ref_name }}
46          git submodule update --init --recursive
47          git checkout -b wasmtime/${{ github.ref }}/${{ github.sha }}
48          export commit_url=${{ github.event.pull_request._links.commits.href }}
49          git config user.name $(curl -sSL $commit_url | jq -r '.[].commit.committer.name' | tail -n 1)
50          git config user.email $(curl -sSL $commit_url | jq -r '.[].commit.committer.email' | tail -n 1)
51          git commit --allow-empty -m "${{ github.event.pull_request._links.comments.href }}"
52          git push origin wasmtime/${{ github.ref }}/${{ github.sha }}
53
54  Performance_Repo_On_Push:
55    name: Benchmark x64 on push Performance repo
56    runs-on: [self-hosted, linux, x64]
57    if: (github.event_name == 'push') && (github.repository == 'bytecodealliance/wasmtime-sightglass-benchmarking')
58    steps:
59      - run: echo "$GITHUB_CONTEXT"
60      - run: echo "${{ github.event.head_commit.message }}"
61      - name: "Build sightglass commit '${{ env.SG_COMMIT }}'"
62        run: |
63          cd ../ && ls -l && rm -rf ./sightglass
64          git clone https://github.com/bytecodealliance/sightglass.git && cd ./sightglass
65          git checkout ${{env.SG_COMMIT}}
66          cargo build --release
67
68      - name: Checkout patch from bytecodealliance/wasmtime (pushed and triggering on this perf repo)
69        uses: actions/checkout@v3
70        with:
71          submodules: true
72          path: wasmtime_commit
73
74      - name: Build patch from bytecodealliance/wasmtime (pushed and triggering on this perf repo)
75        working-directory: ./wasmtime_commit
76        run: |
77          cargo build --release -p wasmtime-bench-api
78          cp target/release/libwasmtime_bench_api.so /tmp/wasmtime_commit.so
79
80      - name: Checkout main from bytecodealliance/wasmtime
81        uses: actions/checkout@v3
82        with:
83          ref: 'main'
84          repository: 'bytecodealliance/wasmtime'
85          submodules: true
86          path: wasmtime_main
87
88      - name: Build main from bytecodealliance/wasmtime
89        working-directory: ./wasmtime_main
90        run: |
91          cargo build --release -p wasmtime-bench-api
92          cp target/release/libwasmtime_bench_api.so /tmp/wasmtime_main.so
93
94      - name: Run performance tests
95        working-directory: ../sightglass
96        run: |
97          cargo run -- \
98          benchmark \
99          --processes 1 \
100          --iterations-per-process 2 \
101          --engine /tmp/wasmtime_main.so \
102          --engine /tmp/wasmtime_commit.so \
103          --output-format csv \
104          --output-file /tmp/results.csv \
105          --raw \
106          -- benchmarks/*/benchmark.wasm
107          ./target/release/sightglass-cli summarize --input-format csv  --output-format csv -f /tmp/results.csv > /tmp/results_summarized.csv
108
109      - name: Setup Python
110        uses: actions/setup-python@v2
111        with:
112          python-version: '3.9'
113
114      - name: Post Process Results
115        run: |
116          pip3 install pandas numpy
117          grep -v "nanoseconds" /tmp/results_summarized.csv > /tmp/results_cycles_summarized.csv
118          sed -i 's/\/tmp\/wasmtime_commit.so/patch/g' /tmp/results_cycles_summarized.csv
119          sed -i 's/\/tmp\/wasmtime_main.so/main/g' /tmp/results_cycles_summarized.csv
120          sed -i 's/benchmarks-next\///g'  /tmp/results_cycles_summarized.csv
121          sed -i 's/\/benchmark.wasm//g'  /tmp/results_cycles_summarized.csv
122          python3 -c "import pandas as pd; pp = pd.read_csv('/tmp/results_cycles_summarized.csv', \
123          usecols=['arch','engine','wasm', 'phase', 'mean'], header=0); \
124          pp_sorted = pp.sort_values(['wasm', 'phase', 'engine'], ascending=True); \
125          pp_pct_changed=pp_sorted.groupby(['wasm','phase'])['mean'].pct_change().reset_index().rename(columns = {'mean':'pct_change'}); \
126          pp_sorted.index.name = 'index'; \
127          pp_sorted_merged=pp_sorted.merge(pp_pct_changed, on='index'); \
128          pp_sorted_merged[pp_sorted_merged['engine'].str.contains('patch')]; \
129          pp_sorted_merged=pp_sorted_merged[pp_sorted_merged['engine'].str.contains('patch')]; \
130          pp_sorted_merged=pp_sorted_merged[['wasm','arch','phase','pct_change']]; \
131          print(pp_sorted_merged.to_string(index=False));" >  /tmp/results_cycles_summarized_sorted2.csv
132          sed -i 's/^/ /' /tmp/results_cycles_summarized_sorted2.csv
133          sed -i 's/ \+/|/g' /tmp/results_cycles_summarized_sorted2.csv
134          sed -i -z 's/\n/|\n/g' /tmp/results_cycles_summarized_sorted2.csv
135          sed -i '2 i\ |-|-|-|-|' /tmp/results_cycles_summarized_sorted2.csv
136          sed -i '/main/d' /tmp/results_cycles_summarized_sorted2.csv
137          sed -i '1 i\Shows pct_change on x64 for the patch if merged compared to current head for main.\n\
138          Pct_change is based on clocktick event cycles where the benchmarks are run with Sightglass. \
139          A negative pct_change means clockticks are expected to be reduced for the benchmark, \
140          for that phase, and by that factor, if the patch were merged (i.e. negative is good).\n' /tmp/results_cycles_summarized_sorted2.csv
141
142      - name: Print Results
143        run: cat /tmp/results_cycles_summarized_sorted2.csv
144
145      - id: get-comment-body
146        name: Create Results Body
147        run: |
148            body="$(cat /tmp/results_cycles_summarized_sorted2.csv)"
149            body="${body//'%'/'%25'}"
150            body="${body//$'\n'/'%0A'}"
151            body="${body//$'\r'/'%0D'}"
152            echo "::set-output name=body::$body"
153
154      - name: Publish Results
155        run: |
156          curl -X POST -H "Accept: application/vnd.github.v3+json" \
157          -H "Authorization: token ${{ secrets.WASMTIME_PUBLISHING_TOKEN }}" \
158          ${{ github.event.head_commit.message }} \
159          -d '{"body": ${{ toJSON(steps.get-comment-body.outputs.body) }}}'
160