Skip to content

Commit 2d17bd6

Browse files
tgregglinlin-s
tgregg
authored and
linlin-s
committed
Adds a system-level IonReader wrapper over the continuable reader.
1 parent 46824d7 commit 2d17bd6

2 files changed

Lines changed: 433 additions & 108 deletions

File tree

‎.github/workflows/ion-java-performance-regression-detector.yml‎

Lines changed: 81 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -3,146 +3,119 @@
33

44
name: Ion Java performance regression detector
55

6-
on: [pull_request]
6+
on:
7+
[pull_request, push]
78

8-
jobs:
9-
PR-Content-Check:
10-
uses: amazon-ion/ion-java/.github/workflows/PR-content-check.yml@master
9+
env:
10+
data_size: '50000'
11+
test_data_id: 'generated-test-data'
1112

12-
detect-regression:
13-
name: Detect Regression
14-
needs: PR-Content-Check
15-
if: ${{ needs.PR-Content-Check.outputs.result == 'pass' }}
13+
jobs:
14+
generate-test-data:
15+
name: Generate Test Data
1616
runs-on: ubuntu-latest
17-
1817
steps:
19-
- name: Set up JDK 1.8
20-
uses: actions/setup-java@v1
21-
with:
22-
java-version: 1.8
23-
24-
- name: Checkout ion-java from the new commit.
25-
uses: actions/checkout@v3
26-
with:
27-
ref: ${{ github.event.pull_request.head.sha }}
28-
path: ion-java-new
29-
30-
- name: Build ion-java from the new commit
31-
run: cd ion-java-new && git submodule init && git submodule update && ./gradlew clean publishToMavenLocal
32-
33-
- name: Checkout ion-java-benchmark-cli
34-
uses: actions/checkout@v3
35-
with:
36-
repository: amazon-ion/ion-java-benchmark-cli
37-
ref: master
38-
path: ion-java-benchmark-cli
39-
40-
- name: Build ion-java-benchmark-cli
41-
run: cd ion-java-benchmark-cli && mvn clean install
42-
4318
- name: Checkout ion-data-generator
44-
uses: actions/checkout@v3
19+
uses: actions/checkout@v4
4520
with:
4621
repository: amazon-ion/ion-data-generator
4722
ref: main
48-
path: ion-data-generator
4923

5024
- name: Build ion-data-generator
51-
run: cd ion-data-generator && mvn clean install
52-
53-
- name: Check the version of ion-java.
54-
run: java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version
25+
run: mvn clean install
5526

56-
- name: Generate test Ion Data
27+
- name: Generate test data
28+
env:
29+
jar_file: target/ion-data-generator-1.0-SNAPSHOT.jar
30+
schema_dir: tst/com/amazon/ion/workflow
5731
run: |
5832
mkdir -p testData
59-
java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/nestedStruct.isl testData/testStructs.10n
60-
java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/nestedList.isl testData/testLists.10n
61-
java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/sexp.isl testData/testSexps.10n
62-
java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/realWorldDataSchema01.isl testData/realWorldData01.10n
63-
java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/realWorldDataSchema02.isl testData/realWorldData02.10n
64-
java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/realWorldDataSchema03.isl testData/realWorldData03.10n
33+
for test in nestedStruct nestedList sexp realWorldDataSchema01 realWorldDataSchema02 realWorldDataSchema03
34+
do
35+
java -jar $jar_file generate -S ${{env.data_size}} --input-ion-schema $schema_dir/${test}.isl testData/${test}.10n
36+
done
6537
6638
- name: Upload test Ion Data to artifacts
67-
uses: actions/upload-artifact@v2
39+
uses: actions/upload-artifact@v3
6840
with:
69-
name: test Ion Data
41+
name: ${{env.test_data_id}}
7042
path: testData
7143

72-
- name: Benchmark ion-java from the new commit
73-
run: |
74-
mkdir -p benchmarkResults
75-
cd ion-java-benchmark-cli && java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar run-suite --test-ion-data /home/runner/work/ion-java/ion-java/testData --benchmark-options-combinations tst/com/amazon/ion/benchmark/optionsCombinations.ion /home/runner/work/ion-java/ion-java/benchmarkResults
44+
detect-regression:
45+
name: Detect Regression
46+
runs-on: ubuntu-latest
47+
needs: generate-test-data
48+
strategy:
49+
matrix:
50+
test-data: ['nestedStruct', 'nestedList', 'sexp', 'realWorldDataSchema01', 'realWorldDataSchema02', 'realWorldDataSchema03']
51+
usage-patterns: [' read --mode AverageTime --forks 3 --warmups 2 --iterations 2 --api streaming ',' read --mode AverageTime --forks 3 --warmups 2 --iterations 2 --api dom ', ' write --mode AverageTime --forks 3 --warmups 2 --iterations 2 --api streaming --io-type buffer --ion-length-preallocation 1 ', ' write --mode AverageTime --forks 3 --warmups 2 --iterations 2 --api dom --io-type buffer --ion-length-preallocation 1 ']
52+
fail-fast: false
53+
54+
steps:
55+
- name: Set up JDK 11
56+
uses: actions/setup-java@v3
57+
with:
58+
distribution: 'corretto'
59+
java-version: 11
7660

77-
- name: Upload benchmark results to artifacts
78-
uses: actions/upload-artifact@v2
61+
- name: Checkout ion-java-benchmark-cli
62+
uses: actions/checkout@v4
7963
with:
80-
name: Benchmark result
81-
path: benchmarkResults
64+
repository: amazon-ion/ion-java-benchmark-cli
65+
ref: cow-optimizeCompare
66+
path: ion-java-benchmark-cli
8267

83-
- name: Clean maven dependencies repository
84-
run : rm -r /home/runner/.m2
68+
- name: Checkout ion-java from the previous commit
69+
uses: actions/checkout@v4
70+
with:
71+
ref: ${{ github.base_ref }}
72+
path: baseline
8573

86-
- name: Checkout the current commit
87-
uses: actions/checkout@v3
74+
- name: Checkout ion-java from the new commit.
75+
uses: actions/checkout@v4
8876
with:
89-
repository: amazon-ion/ion-java
90-
ref: master
91-
path: ion-java
77+
ref: ${{ github.head_ref }}
78+
path: new
9279

93-
- name: Build ion-java from the previous commit
94-
run: cd ion-java && git submodule init && git submodule update && ./gradlew clean publishToMavenLocal
80+
- name: Download test Ion Data from artifacts
81+
uses: actions/download-artifact@v3
82+
with:
83+
name: generated-test-data
84+
path: testData
9585

96-
- name: Build ion-java-benchmark-cli
97-
run: cd ion-java-benchmark-cli && mvn clean install
86+
# Get the benchmark results from the base commit
87+
- name: Build ion-java from the base commit
88+
working-directory: baseline
89+
run: |
90+
git submodule init && git submodule update && ./gradlew clean publishToMavenLocal
91+
92+
- name: Benchmark ion-java from the base commit
93+
working-directory: ion-java-benchmark-cli
94+
run: |
95+
mvn clean install
96+
mkdir /home/runner/work/ion-java/ion-java/benchmarkresults
97+
java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar ${{matrix.usage-patterns}} -r ion -o /home/runner/work/ion-java/ion-java/benchmarkresults/before.ion /home/runner/work/ion-java/ion-java/testData/${{matrix.test-data}}.10n
9898
99-
- name: Check the version of ion-java
100-
run: java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version
10199
102-
- name: Create directories for test data and benchmark results
100+
- name: Build ion-java from the new commit
101+
working-directory: new
103102
run: |
104-
mkdir -p benchmarkResults
105-
mkdir -p testData
103+
git submodule init && git submodule update && ./gradlew clean publishToMavenLocal
106104
107-
- name: Download test Ion Data from artifacts
108-
uses: actions/download-artifact@v2
109-
with:
110-
name: test Ion Data
111-
path: testData
105+
- name: Benchmark ion-java from the new commit
106+
working-directory: ion-java-benchmark-cli
107+
run: |
108+
mvn clean install
109+
java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar ${{matrix.usage-patterns}} -r ion -o /home/runner/work/ion-java/ion-java/benchmarkresults/after.ion /home/runner/work/ion-java/ion-java/testData/${{matrix.test-data}}.10n
112110
113-
- name: Download benchmark results of ion-java from the new commit from artifacts
114-
uses: actions/download-artifact@v2
115-
with:
116-
name: Benchmark result
117-
path: benchmarkResults
111+
#Detect regression
112+
- name: Detect regression
113+
working-directory: ion-java-benchmark-cli
114+
run:
115+
a=$(java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous /home/runner/work/ion-java/ion-java/benchmarkresults/before.ion --benchmark-result-new /home/runner/work/ion-java/ion-java/benchmarkresults/after.ion)
116+
echo ${a} >> $GITHUB_STEP_SUMMARY
118117

119-
- name: Benchmark ion-java from the previous commit and add the generated benchmark results to the existing directories
120-
run: cd ion-java-benchmark-cli && java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar run-suite --test-ion-data /home/runner/work/ion-java/ion-java/testData --benchmark-options-combinations tst/com/amazon/ion/benchmark/optionsCombinations.ion /home/runner/work/ion-java/ion-java/benchmarkResults
121118

122-
- name: Upload new benchmark results directory to artifacts
123-
uses: actions/upload-artifact@v2
124-
with:
125-
name: Benchmark result
126-
path: benchmarkResults
127119

128-
- name: Detect performance regression
129-
id: regression_result
130-
run: |
131-
result=true
132-
cd benchmarkResults && for FILE in *; do message=$(java -jar /home/runner/work/ion-java/ion-java/ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous $FILE/previous.ion --benchmark-result-new $FILE/new.ion $FILE/report.ion | tee /dev/stderr) && if [ "$message" != "no regression detected" ]; then result=false; fi; done
133-
echo "::set-output name=regression-result::$result"
134-
if [ "$result" = "true" ]; then echo "No regression detected!" >> $GITHUB_STEP_SUMMARY; fi
135120

136-
- name: Upload comparison reports to the benchmark results directory
137-
uses: actions/upload-artifact@v2
138-
with:
139-
name: Benchmark result
140-
path: benchmarkResults
141121

142-
- name: Fail the workflow if regression happened
143-
env:
144-
regression_detect: ${{steps.regression_result.outputs.regression-result}}
145-
if: ${{ env.regression_detect == 'false' }}
146-
run: |
147-
cd benchmarkResults && echo "| Benchmark command | GC Allocation Rate | Heap Usage | Speed |" >> $GITHUB_STEP_SUMMARY && echo "| ----------- | ----------- |----------- | ----------- |" >> $GITHUB_STEP_SUMMARY && for FILE in *; do regressionDetection=$(java -jar /home/runner/work/ion-java/ion-java/ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous $FILE/previous.ion --benchmark-result-new $FILE/new.ion $FILE/report.ion) && if [ "$regressionDetection" != "no regression detected" ]; then command=$(echo $FILE | sed "s/_/ /g") && read gc heapUsage speed <<< $( echo ${regressionDetection} | awk -F", " '{print $1" "$2" "$3}' ) && echo "|$command|$gc|$heapUsage|$speed|" >> $GITHUB_STEP_SUMMARY; fi; done
148-
exit 1

0 commit comments

Comments
 (0)