Skip to content

Commit 620228b

Browse files
jobarr-amzntgregg
jobarr-amzn
and
tgregg
committed
Restack 1.1 macros and symbols to meet spec changes (#1013)
* Restack macros and symbols to meet spec changes * Update ion-tests for symbol and macro restack * Fix comment in src/test/java/com/amazon/ion/impl/bin/WriteBufferTest.java Co-authored-by: Tyler Gregg <greggt@amazon.com> --------- Co-authored-by: Tyler Gregg <greggt@amazon.com>
1 parent 7a18550 commit 620228b

10 files changed

Lines changed: 113 additions & 119 deletions

File tree

‎ion-tests‎

‎src/main/java/com/amazon/ion/impl/SystemSymbols_1_1.kt‎

Lines changed: 47 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -24,55 +24,53 @@ enum class SystemSymbols_1_1(val id: Int, val text: String) {
2424
MACRO_TABLE( /* */ 14, "macro_table"),
2525
SYMBOL_TABLE( /* */ 15, "symbol_table"),
2626
MODULE( /* */ 16, "module"),
27-
RETAIN( /* */ 17, "retain"), // https://github.com/amazon-ion/ion-docs/issues/345
28-
EXPORT( /* */ 18, "export"),
29-
CATALOG_KEY( /* */ 19, "catalog_key"), // https://github.com/amazon-ion/ion-docs/issues/345
30-
IMPORT( /* */ 20, "import"),
31-
THE_EMPTY_SYMBOL( /* */ 21, ""),
32-
LITERAL( /* */ 22, "literal"),
33-
IF_NONE( /* */ 23, "if_none"),
34-
IF_SOME( /* */ 24, "if_some"),
35-
IF_SINGLE( /* */ 25, "if_single"),
36-
IF_MULTI( /* */ 26, "if_multi"),
37-
FOR( /* */ 27, "for"),
38-
DEFAULT( /* */ 28, "default"),
39-
VALUES( /* */ 29, "values"),
40-
ANNOTATE( /* */ 30, "annotate"),
41-
MAKE_STRING( /* */ 31, "make_string"),
42-
MAKE_SYMBOL( /* */ 32, "make_symbol"),
43-
MAKE_BLOB( /* */ 33, "make_blob"),
44-
MAKE_DECIMAL( /* */ 34, "make_decimal"),
45-
MAKE_TIMESTAMP( /* */ 35, "make_timestamp"),
46-
MAKE_LIST( /* */ 36, "make_list"),
47-
MAKE_SEXP( /* */ 37, "make_sexp"),
48-
MAKE_STRUCT( /* */ 38, "make_struct"),
49-
PARSE_ION( /* */ 39, "parse_ion"),
50-
REPEAT( /* */ 40, "repeat"),
51-
DELTA( /* */ 41, "delta"),
52-
FLATTEN( /* */ 42, "flatten"),
53-
SUM( /* */ 43, "sum"),
54-
SET_SYMBOLS( /* */ 44, "set_symbols"),
55-
ADD_SYMBOLS( /* */ 45, "add_symbols"),
56-
SET_MACROS( /* */ 46, "set_macros"),
57-
ADD_MACROS( /* */ 47, "add_macros"),
58-
USE( /* */ 48, "use"),
59-
META( /* */ 49, "meta"),
60-
FLEX_SYMBOL( /* */ 50, "flex_symbol"),
61-
FLEX_INT( /* */ 51, "flex_int"),
62-
FLEX_UINT( /* */ 52, "flex_uint"),
63-
UINT8( /* */ 53, "uint8"),
64-
UINT16( /* */ 54, "uint16"),
65-
UINT32( /* */ 55, "uint32"),
66-
UINT64( /* */ 56, "uint64"),
67-
INT8( /* */ 57, "int8"),
68-
INT16( /* */ 58, "int16"),
69-
INT32( /* */ 59, "int32"),
70-
INT64( /* */ 60, "int64"),
71-
FLOAT16( /* */ 61, "float16"),
72-
FLOAT32( /* */ 62, "float32"),
73-
FLOAT64( /* */ 63, "float64"),
74-
NONE( /* */ 64, "none"),
75-
MAKE_FIELD( /* */ 65, "make_field"),
27+
EXPORT( /* */ 17, "export"),
28+
IMPORT( /* */ 18, "import"),
29+
FLEX_SYMBOL( /* */ 19, "flex_symbol"),
30+
FLEX_INT( /* */ 20, "flex_int"),
31+
FLEX_UINT( /* */ 21, "flex_uint"),
32+
UINT8( /* */ 22, "uint8"),
33+
UINT16( /* */ 23, "uint16"),
34+
UINT32( /* */ 24, "uint32"),
35+
UINT64( /* */ 25, "uint64"),
36+
INT8( /* */ 26, "int8"),
37+
INT16( /* */ 27, "int16"),
38+
INT32( /* */ 28, "int32"),
39+
INT64( /* */ 29, "int64"),
40+
FLOAT16( /* */ 30, "float16"),
41+
FLOAT32( /* */ 31, "float32"),
42+
FLOAT64( /* */ 32, "float64"),
43+
EMPTY_TEXT( /* */ 33, ""),
44+
FOR( /* */ 34, "for"),
45+
LITERAL( /* */ 35, "literal"),
46+
IF_NONE( /* */ 36, "if_none"),
47+
IF_SOME( /* */ 37, "if_some"),
48+
IF_SINGLE( /* */ 38, "if_single"),
49+
IF_MULTI( /* */ 39, "if_multi"),
50+
NONE( /* */ 40, "none"),
51+
VALUES( /* */ 41, "values"),
52+
DEFAULT( /* */ 42, "default"),
53+
META( /* */ 43, "meta"),
54+
REPEAT( /* */ 44, "repeat"),
55+
FLATTEN( /* */ 45, "flatten"),
56+
DELTA( /* */ 46, "delta"),
57+
SUM( /* */ 47, "sum"),
58+
ANNOTATE( /* */ 48, "annotate"),
59+
MAKE_STRING( /* */ 49, "make_string"),
60+
MAKE_SYMBOL( /* */ 50, "make_symbol"),
61+
MAKE_DECIMAL( /* */ 51, "make_decimal"),
62+
MAKE_TIMESTAMP( /* */ 52, "make_timestamp"),
63+
MAKE_BLOB( /* */ 53, "make_blob"),
64+
MAKE_LIST( /* */ 54, "make_list"),
65+
MAKE_SEXP( /* */ 55, "make_sexp"),
66+
MAKE_FIELD( /* */ 56, "make_field"),
67+
MAKE_STRUCT( /* */ 57, "make_struct"),
68+
PARSE_ION( /* */ 58, "parse_ion"),
69+
SET_SYMBOLS( /* */ 59, "set_symbols"),
70+
ADD_SYMBOLS( /* */ 60, "add_symbols"),
71+
SET_MACROS( /* */ 61, "set_macros"),
72+
ADD_MACROS( /* */ 62, "add_macros"),
73+
USE( /* */ 63, "use"),
7674
;
7775

7876
val utf8Bytes = text.encodeToByteArray()

‎src/main/java/com/amazon/ion/impl/bin/WriteBuffer.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1536,7 +1536,7 @@ public int writeFlexSym(int sid) {
15361536
*/
15371537
public int writeFlexSym(Utf8StringEncoder.Result text) {
15381538
if (text.getEncodedLength() == 0) {
1539-
return writeFlexSym(SystemSymbols_1_1.THE_EMPTY_SYMBOL);
1539+
return writeFlexSym(SystemSymbols_1_1.EMPTY_TEXT);
15401540
} else {
15411541
int numLengthBytes = writeFlexInt(-text.getEncodedLength());
15421542
writeBytes(text.getBuffer(), 0, text.getEncodedLength());

‎src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -450,20 +450,20 @@ class MacroEvaluator {
450450
@JvmStatic
451451
fun forSystemMacro(macro: SystemMacro): ExpansionKind {
452452
return when (macro) {
453+
SystemMacro.IfNone -> IfNone
454+
SystemMacro.IfSome -> IfSome
455+
SystemMacro.IfSingle -> IfSingle
456+
SystemMacro.IfMulti -> IfMulti
453457
SystemMacro.None -> Values // "none" takes no args, so we can treat it as an empty "values" expansion
454458
SystemMacro.Values -> Values
455459
SystemMacro.Annotate -> Annotate
456460
SystemMacro.MakeString -> MakeString
457461
SystemMacro.MakeSymbol -> MakeSymbol
458-
SystemMacro.MakeBlob -> MakeBlob
459462
SystemMacro.MakeDecimal -> MakeDecimal
460463
SystemMacro.MakeTimestamp -> MakeTimestamp
461-
SystemMacro.IfNone -> IfNone
462-
SystemMacro.IfSome -> IfSome
463-
SystemMacro.IfSingle -> IfSingle
464-
SystemMacro.IfMulti -> IfMulti
465-
SystemMacro.Repeat -> Repeat
464+
SystemMacro.MakeBlob -> MakeBlob
466465
SystemMacro.MakeField -> MakeField
466+
SystemMacro.Repeat -> Repeat
467467
SystemMacro.Sum -> Sum
468468
SystemMacro.Delta -> Delta
469469
else -> if (macro.body != null) {

‎src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt‎

Lines changed: 33 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,24 @@ enum class SystemMacro(
2929
// The real macros
3030
None(0, NONE, emptyList()),
3131
Values(1, VALUES, listOf(zeroToManyTagged("values"))),
32-
Annotate(2, ANNOTATE, listOf(zeroToManyTagged("ann"), exactlyOneTagged("value"))),
33-
MakeString(3, MAKE_STRING, listOf(zeroToManyTagged("text"))),
34-
MakeSymbol(4, MAKE_SYMBOL, listOf(zeroToManyTagged("text"))),
35-
MakeBlob(5, MAKE_BLOB, listOf(zeroToManyTagged("bytes"))),
36-
MakeDecimal(6, MAKE_DECIMAL, listOf(exactlyOneTagged("coefficient"), exactlyOneTagged("exponent"))),
32+
Default(
33+
2, DEFAULT, listOf(zeroToManyTagged("expr"), zeroToManyTagged("default_expr")),
34+
templateBody {
35+
macro(IfNone) { variable(0); variable(1); variable(0) }
36+
}
37+
),
38+
Meta(3, META, listOf(zeroToManyTagged("values")), templateBody { macro(None) {} }),
39+
Repeat(4, REPEAT, listOf(exactlyOneTagged("n"), zeroToManyTagged("value"))),
40+
Flatten(5, FLATTEN, listOf(zeroToManyTagged("values")), null), // TODO: flatten
41+
Delta(6, DELTA, listOf(zeroToManyTagged("deltas"))),
42+
Sum(7, SUM, listOf(exactlyOneTagged("a"), exactlyOneTagged("b"))),
43+
44+
Annotate(8, ANNOTATE, listOf(zeroToManyTagged("ann"), exactlyOneTagged("value"))),
45+
MakeString(9, MAKE_STRING, listOf(zeroToManyTagged("text"))),
46+
MakeSymbol(10, MAKE_SYMBOL, listOf(zeroToManyTagged("text"))),
47+
MakeDecimal(11, MAKE_DECIMAL, listOf(exactlyOneTagged("coefficient"), exactlyOneTagged("exponent"))),
3748
MakeTimestamp(
38-
7, MAKE_TIMESTAMP,
49+
12, MAKE_TIMESTAMP,
3950
listOf(
4051
exactlyOneTagged("year"),
4152
zeroOrOneTagged("month"),
@@ -46,9 +57,17 @@ enum class SystemMacro(
4657
zeroOrOneTagged("offset_minutes"),
4758
)
4859
),
49-
// TODO: make_list
50-
// TODO: make_sexp
51-
// TODO: make_struct
60+
MakeBlob(13, MAKE_BLOB, listOf(zeroToManyTagged("bytes"))),
61+
MakeList(14, MAKE_LIST, listOf(zeroToManyTagged("sequences")), null), // TODO: make_list
62+
MakeSExp(15, MAKE_SEXP, listOf(zeroToManyTagged("sequences")), null), // TODO: make_sexp
63+
MakeField(
64+
16, MAKE_FIELD,
65+
listOf(
66+
Macro.Parameter("field_name", Macro.ParameterEncoding.FlexSym, Macro.ParameterCardinality.ExactlyOne), exactlyOneTagged("value")
67+
)
68+
),
69+
MakeStruct(17, MAKE_STRUCT, listOf(zeroToManyTagged("structs")), null), // TODO: make_struct
70+
ParseIon(18, PARSE_ION, listOf(zeroToManyTagged("data")), null), // TODO: parse_ion
5271

5372
/**
5473
* ```ion
@@ -60,7 +79,7 @@ enum class SystemMacro(
6079
* ```
6180
*/
6281
SetSymbols(
63-
11, SET_SYMBOLS, listOf(zeroToManyTagged("symbols")),
82+
19, SET_SYMBOLS, listOf(zeroToManyTagged("symbols")),
6483
templateBody {
6584
annotated(ION, ::sexp) {
6685
symbol(MODULE)
@@ -87,7 +106,7 @@ enum class SystemMacro(
87106
* ```
88107
*/
89108
AddSymbols(
90-
12, ADD_SYMBOLS, listOf(zeroToManyTagged("symbols")),
109+
20, ADD_SYMBOLS, listOf(zeroToManyTagged("symbols")),
91110
templateBody {
92111
annotated(ION, ::sexp) {
93112
symbol(MODULE)
@@ -115,7 +134,7 @@ enum class SystemMacro(
115134
* ```
116135
*/
117136
SetMacros(
118-
13, SET_MACROS, listOf(zeroToManyTagged("macros")),
137+
21, SET_MACROS, listOf(zeroToManyTagged("macros")),
119138
templateBody {
120139
annotated(ION, ::sexp) {
121140
symbol(MODULE)
@@ -142,7 +161,7 @@ enum class SystemMacro(
142161
* ```
143162
*/
144163
AddMacros(
145-
14, ADD_MACROS, listOf(zeroToManyTagged("macros")),
164+
22, ADD_MACROS, listOf(zeroToManyTagged("macros")),
146165
templateBody {
147166
annotated(ION, ::sexp) {
148167
symbol(MODULE)
@@ -171,7 +190,7 @@ enum class SystemMacro(
171190
* ```
172191
*/
173192
Use(
174-
15, USE, listOf(exactlyOneTagged("catalog_key"), zeroOrOneTagged("version")),
193+
23, USE, listOf(exactlyOneTagged("catalog_key"), zeroOrOneTagged("version")),
175194
templateBody {
176195
val theModule = _Private_Utils.newSymbolToken("the_module")
177196
annotated(ION, ::sexp) {
@@ -201,29 +220,6 @@ enum class SystemMacro(
201220
}
202221
}
203222
),
204-
// TODO: parse_ion
205-
Repeat(17, REPEAT, listOf(exactlyOneTagged("n"), zeroToManyTagged("value"))),
206-
Delta(18, DELTA, listOf(zeroToManyTagged("deltas"))),
207-
// TODO: flatten
208-
Sum(20, SUM, listOf(exactlyOneTagged("a"), exactlyOneTagged("b"))),
209-
Comment(21, META, listOf(zeroToManyTagged("values")), templateBody { macro(None) {} }),
210-
MakeField(
211-
22, MAKE_FIELD,
212-
listOf(
213-
Macro.Parameter("field_name", Macro.ParameterEncoding.FlexSym, Macro.ParameterCardinality.ExactlyOne), exactlyOneTagged("value")
214-
)
215-
),
216-
217-
Default(
218-
23, DEFAULT, listOf(zeroToManyTagged("expr"), zeroToManyTagged("default_expr")),
219-
templateBody {
220-
macro(IfNone) {
221-
variable(0)
222-
variable(1)
223-
variable(0)
224-
}
225-
}
226-
),
227223
;
228224

229225
val macroName: String get() = this.systemSymbol.text

‎src/test/java/com/amazon/ion/impl/IonReaderContinuableCoreBinaryTest.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1069,7 +1069,7 @@ public void taglessCompactSymbols(boolean constructFromBytes) throws Exception {
10691069
0x00, // User macro ID 0
10701070
0xF9, 0x6E, 0x61, 0x6D, 0x65, // interpreted as compact symbol (FlexSym with inline text "name")
10711071
0x09, // interpreted as compact symbol (FlexSym with SID 4)
1072-
0x01, 0x75 // interpreted as compact symbol (special FlexSym)
1072+
0x01, 0x81 // interpreted as compact symbol (special FlexSym)
10731073
));
10741074
try (IonReaderContinuableCoreBinary reader = initializeReader(constructFromBytes, data)) {
10751075
assertSequence(
@@ -1090,7 +1090,7 @@ public void taglessCompactSymbols(boolean constructFromBytes) throws Exception {
10901090
@ValueSource(booleans = {true, false})
10911091
public void addSymbolsSystemMacro(boolean constructFromBytes) throws Exception {
10921092
byte[] data = withIvm(1, bytes(
1093-
0xEF, 0x0C, // system macro add_symbols
1093+
0xEF, 0x14, // system macro add_symbols
10941094
0x02, // AEB: 0b------aa; a=10, expression group
10951095
0x01, // FlexInt 0, a delimited expression group
10961096
0x93, 0x61, 0x62, 0x63, // 3-byte string, utf-8 "abc"
@@ -1181,13 +1181,13 @@ private void assertMacroAwareTranscribeProducesEquivalentStream(byte[] data, boo
11811181
@ValueSource(booleans = {true, false})
11821182
public void encodingLevelTranscodeOfSystemMacroInvocation(boolean constructFromBytes) throws Exception {
11831183
byte[] data = withIvm(1, bytes(
1184-
0xEF, 0x0C, // system macro add_symbols
1184+
0xEF, 0x14, // system macro add_symbols
11851185
0x02, // AEB: 0b------aa; a=10, expression group
11861186
0x01, // FlexInt 0, a delimited expression group
11871187
0x93, 0x61, 0x62, 0x63, // 3-byte string, utf-8 "abc"
11881188
0xF0, // delimited end... of expression group
11891189
0xE1, // SID single byte
1190-
0x42 // SID $66
1190+
0x40 // SID $64
11911191
));
11921192
assertMacroAwareTranscribeProducesEquivalentStream(data, constructFromBytes);
11931193
}

‎src/test/java/com/amazon/ion/impl/IonReaderContinuableTopLevelBinaryTest.java‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5062,15 +5062,15 @@ private void readAnnotationsWithSpecialFlexSyms_1_1(boolean constructFromBytes,
50625062
@ParameterizedTest
50635063
@ValueSource(strings = {
50645064
// Minimal representations
5065-
"E7 01 75 60 | One empty-text annotation; value int 0 \n" +
5065+
"E7 01 81 60 | One empty-text annotation; value int 0 \n" +
50665066
"E7 01 60 60 | One SID 0 annotation; value int 0 \n" +
5067-
"E8 01 75 01 60 60 | Two annotations: empty text, SID 0; value int 0 \n" +
5068-
"E9 09 01 60 01 75 60 | Variable length = 4 annotations: SID 0, empty text; value int 0 \n",
5067+
"E8 01 81 01 60 60 | Two annotations: empty text, SID 0; value int 0 \n" +
5068+
"E9 09 01 60 01 81 60 | Variable length = 4 annotations: SID 0, empty text; value int 0 \n",
50695069
// Overpadded representations
5070-
"E7 02 00 75 60 | One overpadded empty-text annotation; value int 0 \n" +
5070+
"E7 02 00 81 60 | One overpadded empty-text annotation; value int 0 \n" +
50715071
"E7 04 00 00 60 60 | One overpadded SID 0 annotation; value int 0 \n" +
5072-
"E8 08 00 00 00 75 02 00 60 60 | Two overpadded annotations: empty text, SID 0; value int 0 \n" +
5073-
"E9 90 00 00 00 00 01 60 01 75 60 | Variable overpadded length = 4 annotations: SID 0, empty text; value int 0 \n"
5072+
"E8 08 00 00 00 81 02 00 60 60 | Two overpadded annotations: empty text, SID 0; value int 0 \n" +
5073+
"E9 90 00 00 00 00 01 60 01 81 60 | Variable overpadded length = 4 annotations: SID 0, empty text; value int 0 \n"
50745074
})
50755075
public void readAnnotationsWithSpecialFlexSyms_1_1(String inputBytes) throws Exception {
50765076
readAnnotationsWithSpecialFlexSyms_1_1(true, inputBytes);
@@ -5308,17 +5308,17 @@ public void assertStructWithEmptyInlineFieldNamesCorrectlyParsed(boolean constru
53085308
// Empty field name in fixed-length SID struct
53095309
"D4 | Struct Length = 4 \n" +
53105310
"01 | switch to FlexSym encoding \n" +
5311-
"01 75 | FlexSym empty text \n" +
5311+
"01 81 | FlexSym empty text \n" +
53125312
"6F | false",
53135313
// Empty field name in variable-length SID struct
53145314
"FD | Variable length SID struct \n" +
53155315
"09 | Length = 4 \n" +
53165316
"01 | switch to FlexSym encoding \n" +
5317-
"01 75 | FlexSym empty text \n" +
5317+
"01 81 | FlexSym empty text \n" +
53185318
"6F | false",
53195319
// Empty field name in delimited struct
53205320
"F3 | Delimited struct \n" +
5321-
"01 75 | FlexSym empty text \n" +
5321+
"01 81 | FlexSym empty text \n" +
53225322
"6F | false \n" +
53235323
"01 F0 | End delimited struct"
53245324
})
@@ -5990,13 +5990,13 @@ private void invokeValuesUsingSystemMacroOpcodeHelper(boolean constructFromBytes
59905990
public void addSymbolsSystemMacro(boolean constructFromBytes) throws Exception {
59915991
int[] data = new int[] {
59925992
0xE0, 0x01, 0x01, 0xEA, // Ion 1.1 IVM
5993-
0xEF, 0x0C, // system macro add_symbols
5993+
0xEF, 0x14, // system macro add_symbols
59945994
0x02, // AEB: 0b------aa; a=10, expression group
59955995
0x01, // FlexInt 0, a delimited expression group
59965996
0x93, 0x61, 0x62, 0x63, // 3-byte string, utf-8 "abc"
59975997
0xF0, // delimited end... of expression group
59985998
0xE1, // SID single byte
5999-
0x42 // SID $66
5999+
0x40 // SID $64
60006000
};
60016001
try (IonReader reader = readerFor(constructFromBytes,data)) {
60026002
assertEquals(IonType.SYMBOL, reader.next());
@@ -6008,13 +6008,13 @@ public void addSymbolsSystemMacro(boolean constructFromBytes) throws Exception {
60086008
public void addSymbolsSystemMacroWhenNotEntirelyBuffered() throws Exception {
60096009
int[] data = new int[] {
60106010
/* 0 - 3 */ // 0xEA, 0x01, 0x01, 0xE0, // implicitly provided by BinaryIonAppender
6011-
/* 4 - 5 */ 0xEF, 0x0C, // system macro add_symbols
6011+
/* 4 - 5 */ 0xEF, 0x14, // system macro add_symbols
60126012
/* 6 - 7 */ 0x02, 0x01, // AEB: 0b------aa; a=10, FlexInt 0, a delimited expression group
60136013
/* 8 - 12 */ 0x93, 0x66, 0x6F, 0x72, // "for"
60146014
/* 13 - 16 */ 0x93, 0x66, 0x75, 0x72, // "fur"
60156015
/* 17 - 21 */ 0x94, 0x66, 0x6F, 0x75, 0x72, // "four"
60166016
/* 22 - 22 */ 0xF0, // delimited end... of expression group
6017-
/* 23 - 24 */ 0xE1, 0x41 + 3, // SID single byte ${usid 1} => "four"
6017+
/* 23 - 24 */ 0xE1, 0x3F + 3, // SID single byte ${usid 1} => "four"
60186018
};
60196019
byte[] bytes = new TestUtils.BinaryIonAppender(1).append(data).toByteArray();
60206020
totalBytesInStream = bytes.length;

0 commit comments

Comments
 (0)