Skip to content

Commit 8cb7ee4

Browse files
author
Chen
committed
Migrate to MacOS, decrease threshold to 0.25, iterations/warmups to 1k
1 parent 25a5464 commit 8cb7ee4

3 files changed

Lines changed: 37 additions & 36 deletions

File tree

‎.github/workflows/performance-regression.yml‎

Lines changed: 35 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ jobs:
1212
name: Detect Regression
1313
needs: PR-Content-Check
1414
if: ${{ needs.PR-Content-Check.outputs.result == 'pass' }}
15-
runs-on: ubuntu-latest
15+
runs-on: macos-latest # ubuntu-latest
1616
strategy:
1717
matrix:
1818
python-version: ['3.9', '3.10', 'pypy-3.7', 'pypy-3.8']
@@ -66,23 +66,23 @@ jobs:
6666
- name: Running performance benchmark
6767
working-directory: ./ion-python-current
6868
env:
69-
warmups: 150
70-
iterations: 150
69+
warmups: 1000
70+
iterations: 1000
7171
run: |
7272
. venv/bin/activate
73-
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/testStructs_read/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/testStructs.10n
74-
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/testLists_read/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/testLists.10n
75-
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/testSexps_read/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/testSexps.10n
76-
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/realWorldData01_read/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/realWorldData01.10n
77-
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/realWorldData02_read/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/realWorldData02.10n
78-
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/realWorldData03_read/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/realWorldData03.10n
79-
80-
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/testStructs_write/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/testStructs.10n
81-
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/testLists_write/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/testLists.10n
82-
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/testSexps_write/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/testSexps.10n
83-
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/realWorldData01_write/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/realWorldData01.10n
84-
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/realWorldData02_write/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/realWorldData02.10n
85-
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/realWorldData03_write/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/realWorldData03.10n
73+
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/testStructs_read/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/testStructs.10n
74+
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/testLists_read/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/testLists.10n
75+
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/testSexps_read/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/testSexps.10n
76+
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/realWorldData01_read/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/realWorldData01.10n
77+
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/realWorldData02_read/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/realWorldData02.10n
78+
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/realWorldData03_read/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/realWorldData03.10n
79+
80+
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/testStructs_write/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/testStructs.10n
81+
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/testLists_write/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/testLists.10n
82+
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/testSexps_write/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/testSexps.10n
83+
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/realWorldData01_write/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/realWorldData01.10n
84+
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/realWorldData02_write/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/realWorldData02.10n
85+
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/realWorldData03_write/new --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/realWorldData03.10n
8686
8787
# Generates performance results for the previous commit
8888
- name: Checkout the master branch of the Ion Python
@@ -101,23 +101,23 @@ jobs:
101101
- name: Running performance benchmark
102102
working-directory: ./ion-python-master
103103
env:
104-
warmups: 150
105-
iterations: 150
104+
warmups: 1000
105+
iterations: 1000
106106
run: |
107107
. venv/bin/activate
108-
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/testStructs_read/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/testStructs.10n
109-
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/testLists_read/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/testLists.10n
110-
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/testSexps_read/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/testSexps.10n
111-
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/realWorldData01_read/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/realWorldData01.10n
112-
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/realWorldData02_read/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/realWorldData02.10n
113-
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/realWorldData03_read/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/realWorldData03.10n
114-
115-
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/testStructs_write/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/testStructs.10n
116-
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/testLists_write/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/testLists.10n
117-
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/testSexps_write/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/testSexps.10n
118-
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/realWorldData01_write/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/realWorldData01.10n
119-
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/realWorldData02_write/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/realWorldData02.10n
120-
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /home/runner/work/ion-python/ion-python/benchmarkResults/realWorldData03_write/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /home/runner/work/ion-python/ion-python/testData/realWorldData03.10n
108+
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/testStructs_read/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/testStructs.10n
109+
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/testLists_read/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/testLists.10n
110+
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/testSexps_read/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/testSexps.10n
111+
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/realWorldData01_read/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/realWorldData01.10n
112+
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/realWorldData02_read/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/realWorldData02.10n
113+
python amazon/ionbenchmark/ion_benchmark_cli.py read -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/realWorldData03_read/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/realWorldData03.10n
114+
115+
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/testStructs_write/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/testStructs.10n
116+
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/testLists_write/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/testLists.10n
117+
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/testSexps_write/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/testSexps.10n
118+
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/realWorldData01_write/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/realWorldData01.10n
119+
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/realWorldData02_write/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/realWorldData02.10n
120+
python amazon/ionbenchmark/ion_benchmark_cli.py write -w $warmups --iterations $iterations -o /Users/runner/work/ion-python/ion-python/benchmarkResults/realWorldData03_write/previous --io-type file --io-type buffer --format ion_text --format ion_binary --api load_dump /Users/runner/work/ion-python/ion-python/testData/realWorldData03.10n
121121
122122
# Upload resource, results and report
123123
- name: Upload new benchmark results directory to artifacts
@@ -136,9 +136,9 @@ jobs:
136136
- name: Detect performance regression
137137
id: regression_result
138138
run: |
139-
. /home/runner/work/ion-python/ion-python/ion-python-current/venv/bin/activate
139+
. /Users/runner/work/ion-python/ion-python/ion-python-current/venv/bin/activate
140140
result=true
141-
cd benchmarkResults && for FILE in *; do message=$(python /home/runner/work/ion-python/ion-python/ion-python-current/amazon/ionbenchmark/ion_benchmark_cli.py compare --benchmark-result-previous $FILE/previous --benchmark-result-new $FILE/new $FILE/report | tee /dev/stderr) && if [ "$message" != "no regression detected" ]; then result=false; fi; done
141+
cd benchmarkResults && for FILE in *; do message=$(python /Users/runner/work/ion-python/ion-python/ion-python-current/amazon/ionbenchmark/ion_benchmark_cli.py compare --benchmark-result-previous $FILE/previous --benchmark-result-new $FILE/new $FILE/report | tee /dev/stderr) && if [ "$message" != "no regression detected" ]; then result=false; fi; done
142142
echo "::set-output name=regression-result::$result"
143143
if [ "$result" = "true" ]; then echo "No regression detected!" >> $GITHUB_STEP_SUMMARY; fi
144144
@@ -153,6 +153,6 @@ jobs:
153153
regression_detect: ${{steps.regression_result.outputs.regression-result}}
154154
if: ${{ env.regression_detect == 'false' }}
155155
run: |
156-
. /home/runner/work/ion-python/ion-python/ion-python-current/venv/bin/activate
157-
cd benchmarkResults && echo "**Below files are detected performance regression, please download the benchmark result(s) to see detail:**" >> $GITHUB_STEP_SUMMARY && for FILE in *; do regressionDetection=$(python /home/runner/work/ion-python/ion-python/ion-python-current/amazon/ionbenchmark/ion_benchmark_cli.py compare --benchmark-result-previous $FILE/previous --benchmark-result-new $FILE/new $FILE/report | tee /dev/stderr) && if [ "$regressionDetection" != "no regression detected" ]; then echo "$FILE" >> $GITHUB_STEP_SUMMARY; fi; done
156+
. /Users/runner/work/ion-python/ion-python/ion-python-current/venv/bin/activate
157+
cd benchmarkResults && echo "**Below files are detected performance regression, please download the benchmark result(s) to see detail:**" >> $GITHUB_STEP_SUMMARY && for FILE in *; do regressionDetection=$(python /Users/runner/work/ion-python/ion-python/ion-python-current/amazon/ionbenchmark/ion_benchmark_cli.py compare --benchmark-result-previous $FILE/previous --benchmark-result-new $FILE/new $FILE/report | tee /dev/stderr) && if [ "$regressionDetection" != "no regression detected" ]; then echo "$FILE" >> $GITHUB_STEP_SUMMARY; fi; done
158158
exit 1

‎amazon/ion/ioncmodule.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ static int offset_seconds(PyObject* timedelta) {
134134
return seconds;
135135
}
136136

137+
137138
/*
138139
* Returns the ion type of an object as an int
139140
*

0 commit comments

Comments
 (0)