Skip to content

Fix the problem of logical type not propagating properly #117

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Karolina-Bogacka merged 4 commits into from
May 13, 2025
Merged

Fix the problem of logical type not propagating properly #117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
94f8994
Add tests
Karolina-Bogacka May 13, 2025
bcfa637
Fix the bug by avoiding the autodetect
Karolina-Bogacka May 13, 2025
faee6c5
Add more tests
Karolina-Bogacka May 13, 2025
12d29c5
Fix review comments
Karolina-Bogacka May 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 44 additions & 16 deletions src/main/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJelly.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ import eu.neverblink.jelly.cli.*
import eu.neverblink.jelly.cli.command.rdf.util.*
import eu.neverblink.jelly.cli.command.rdf.util.RdfFormat.*
import eu.neverblink.jelly.cli.util.jena.riot.JellyStreamWriterGraphs
import eu.ostrzyciel.jelly.convert.jena.riot.{JellyFormatVariant, JellyLanguage}
import eu.ostrzyciel.jelly.core.proto.v1.{LogicalStreamType, RdfStreamFrame}
import eu.ostrzyciel.jelly.convert.jena.riot.{JellyFormatVariant, JellyLanguage, JellyStreamWriter}
import eu.ostrzyciel.jelly.core.proto.v1.{LogicalStreamType, RdfStreamFrame, RdfStreamOptions}
import org.apache.jena.riot.system.StreamRDFWriter
import org.apache.jena.riot.{Lang, RDFParser, RIOT}

Expand Down Expand Up @@ -120,22 +120,39 @@ object RdfToJelly extends RdfSerDesCommand[RdfToJellyOptions, RdfFormat.Readable
)
else
// TRIPLES or QUADS
val writerContext = RIOT.getContext.copy()
.set(
JellyLanguage.SYMBOL_STREAM_OPTIONS,
jellyOpt,
if jellyOpt.physicalType.isUnspecified then
if !isQuietMode && isLogicalGrouped(jellyOpt) then
printLine(
"WARNING: Logical type setting ignored because physical type is not set. " +
"Set the physical type to properly pass on the logical type." +
"Use --quiet to silence this warning.",
Comment thread
Ostrzyciel marked this conversation as resolved.
true,
)
val writerContext = RIOT.getContext.copy()
.set(
JellyLanguage.SYMBOL_STREAM_OPTIONS,
jellyOpt,
)
.set(JellyLanguage.SYMBOL_FRAME_SIZE, getOptions.rowsPerFrame)
.set(
JellyLanguage.SYMBOL_ENABLE_NAMESPACE_DECLARATIONS,
getOptions.enableNamespaceDeclarations,
).set(JellyLanguage.SYMBOL_DELIMITED_OUTPUT, getOptions.delimited)
StreamRDFWriter.getWriterStream(
outputStream,
JellyLanguage.JELLY,
writerContext,
)
.set(JellyLanguage.SYMBOL_FRAME_SIZE, getOptions.rowsPerFrame)
.set(
JellyLanguage.SYMBOL_ENABLE_NAMESPACE_DECLARATIONS,
getOptions.enableNamespaceDeclarations,
else
// If the physical type is specified, we can just construct the writer
val variant = JellyFormatVariant(
opt = jellyOpt,
frameSize = getOptions.rowsPerFrame,
enableNamespaceDeclarations = getOptions.enableNamespaceDeclarations,
delimited = getOptions.delimited,
)
.set(JellyLanguage.SYMBOL_DELIMITED_OUTPUT, getOptions.delimited)
StreamRDFWriter.getWriterStream(
outputStream,
JellyLanguage.JELLY,
writerContext,
)
JellyStreamWriter(variant, outputStream)

RDFParser.source(inputStream).lang(jenaLang).parse(jellyWriter)

/** Convert Jelly text to Jelly binary.
Expand Down Expand Up @@ -163,6 +180,17 @@ object RdfToJelly extends RdfSerDesCommand[RdfToJellyOptions, RdfFormat.Readable
}
}

/** Check if the logical type is defined and grouped.
* @param jellyOpt
* the Jelly options
* @return
* true if the logical type is specified and expects framing
*/
private def isLogicalGrouped(
jellyOpt: RdfStreamOptions,
): Boolean =
!(jellyOpt.logicalType.isFlatQuads || jellyOpt.logicalType.isFlatTriples || jellyOpt.logicalType.isUnspecified)
Comment thread
Karolina-Bogacka marked this conversation as resolved.

/** Iterate over a Jelly text stream and return the frames as strings to be parsed.
* @param reader
* the reader to read from
Expand Down
127 changes: 115 additions & 12 deletions src/test/scala/eu/neverblink/jelly/cli/command/rdf/RdfToJellySpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import eu.neverblink.jelly.cli.command.helpers.{DataGenHelper, TestFixtureHelper
import eu.neverblink.jelly.cli.command.rdf.util.RdfFormat
import eu.neverblink.jelly.cli.*
import eu.ostrzyciel.jelly.convert.jena.riot.JellyLanguage
import eu.ostrzyciel.jelly.core.proto.v1.{LogicalStreamType, RdfStreamFrame}
import eu.ostrzyciel.jelly.core.proto.v1.{LogicalStreamType, PhysicalStreamType, RdfStreamFrame}
import eu.ostrzyciel.jelly.core.{IoUtils, JellyOptions}
import org.apache.jena.rdf.model.{Model, ModelFactory}
import org.apache.jena.riot.{RDFLanguages, RDFParser}
Expand Down Expand Up @@ -42,7 +42,7 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers:
}

"rdf to-jelly command" should {
"handle conversion of NTriples to Jelly" when {
"handle conversion of NQuads to Jelly" when {
"a file to output stream" in withFullJenaFile { f =>
val (out, err) =
RdfToJelly.runTestCommand(List("rdf", "to-jelly", f))
Expand Down Expand Up @@ -227,6 +227,37 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers:
}
}

"a file to file, physical type set to QUADS, logical type to DATASET STREAM" in withFullJenaFile {
f =>
withEmptyJellyFile { j =>
val (out, err) =
RdfToJelly.runTestCommand(
List(
"rdf",
"to-jelly",
f,
"--opt.physical-type=QUADS",
"--opt.logical-type=DATASETS",
"--to",
j,
),
)
val content = translateJellyBack(new FileInputStream(j))
content.containsAll(DataGenHelper.generateTripleModel(testCardinality).listStatements())
val frames = readJellyFile(new FileInputStream(j))
val opts = frames.head.rows.head.row.options
opts.streamName should be("")
opts.generalizedStatements should be(true)
opts.rdfStar should be(true)
opts.maxNameTableSize should be(JellyOptions.bigStrict.maxNameTableSize)
opts.maxPrefixTableSize should be(JellyOptions.bigStrict.maxPrefixTableSize)
opts.maxDatatypeTableSize should be(JellyOptions.bigStrict.maxDatatypeTableSize)
opts.physicalType should be(PhysicalStreamType.QUADS)
opts.logicalType should be(LogicalStreamType.DATASETS)
opts.version should be(1)
}
}

"a file to file, lowered number of rows per frame" in withFullJenaFile { f =>
withEmptyJellyFile { j =>
val (out, err) =
Expand Down Expand Up @@ -297,16 +328,88 @@ class RdfToJellySpec extends AnyWordSpec with TestFixtureHelper with Matchers:
}

"handle conversion of other formats to Jelly" when {
"NTriples" in {
val input = DataGenHelper.generateJenaInputStream(testCardinality, RDFLanguages.NTRIPLES)
RdfToJelly.setStdIn(input)
val (out, err) =
RdfToJelly.runTestCommand(
List("rdf", "to-jelly", "--in-format", RdfFormat.NTriples.cliOptions.head),
)
val newIn = new ByteArrayInputStream(RdfToJelly.getOutBytes)
val content = translateJellyBack(newIn)
content.containsAll(DataGenHelper.generateTripleModel(testCardinality).listStatements())
"NTriples" when {
"base functionality expected" in {
val input = DataGenHelper.generateJenaInputStream(testCardinality, RDFLanguages.NTRIPLES)
RdfToJelly.setStdIn(input)
val (out, err) =
RdfToJelly.runTestCommand(
List("rdf", "to-jelly", "--in-format", RdfFormat.NTriples.cliOptions.head),
)
val newIn = new ByteArrayInputStream(RdfToJelly.getOutBytes)
val content = translateJellyBack(newIn)
content.containsAll(DataGenHelper.generateTripleModel(testCardinality).listStatements())
}
"a file to file, physical type set to TRIPLES, logical type to GRAPHS" in withFullJenaFile(
testCode = { f =>
withEmptyJellyFile { j =>
val (out, err) =
RdfToJelly.runTestCommand(
List(
"rdf",
"to-jelly",
f,
"--opt.physical-type=TRIPLES",
"--opt.logical-type=GRAPHS",
"--to",
j,
),
)
val content = translateJellyBack(new FileInputStream(j))
content.containsAll(
DataGenHelper.generateTripleModel(testCardinality).listStatements(),
)
val frames = readJellyFile(new FileInputStream(j))
val opts = frames.head.rows.head.row.options
opts.streamName should be("")
opts.generalizedStatements should be(true)
opts.rdfStar should be(true)
opts.maxNameTableSize should be(JellyOptions.bigStrict.maxNameTableSize)
opts.maxPrefixTableSize should be(JellyOptions.bigStrict.maxPrefixTableSize)
opts.maxDatatypeTableSize should be(JellyOptions.bigStrict.maxDatatypeTableSize)
opts.physicalType should be(PhysicalStreamType.TRIPLES)
opts.logicalType should be(LogicalStreamType.GRAPHS)
opts.version should be(1)
}
},
jenaLang = RDFLanguages.NTRIPLES,
)

"a file to file, physical type unspecified, logical type set to GRAPHS" in withFullJenaFile(
testCode = { f =>
withEmptyJellyFile { j =>
val (out, err) =
RdfToJelly.runTestCommand(
List(
"rdf",
"to-jelly",
f,
"--opt.logical-type=GRAPHS",
"--to",
j,
),
)
val content = translateJellyBack(new FileInputStream(j))
content.containsAll(
DataGenHelper.generateTripleModel(testCardinality).listStatements(),
)
val frames = readJellyFile(new FileInputStream(j))
val opts = frames.head.rows.head.row.options
opts.streamName should be("")
opts.generalizedStatements should be(true)
opts.rdfStar should be(true)
opts.maxNameTableSize should be(JellyOptions.bigStrict.maxNameTableSize)
opts.maxPrefixTableSize should be(JellyOptions.bigStrict.maxPrefixTableSize)
opts.maxDatatypeTableSize should be(JellyOptions.bigStrict.maxDatatypeTableSize)
opts.logicalType should be(LogicalStreamType.FLAT_TRIPLES)
opts.version should be(1)
RdfToJelly.getErrString should include(
"WARNING: Logical type setting ignored because physical type is not set.",
)
}
},
jenaLang = RDFLanguages.NTRIPLES,
)
}
"Turtle" in {
val input = DataGenHelper.generateJenaInputStream(testCardinality, RDFLanguages.TURTLE)
Expand Down