Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ object RdfInspect extends JellyCommand[RdfInspectOptions]:
frame: RdfStreamFrame,
frameIndex: Int,
): FrameInfo =
val metrics = new FrameInfo(frameIndex)
val metrics = new FrameInfo(frameIndex, frame.metadata)
frame.rows.foreach(r => metricsForRow(r, metrics))
metrics

Expand Down
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
package eu.neverblink.jelly.cli.command.rdf.util

import eu.neverblink.jelly.cli.util.io.YamlDocBuilder.*
import com.google.protobuf.ByteString
import eu.neverblink.jelly.cli.util.io.YamlDocBuilder
import eu.neverblink.jelly.cli.util.io.YamlDocBuilder.*
import eu.ostrzyciel.jelly.core.proto.v1.RdfStreamOptions

import java.io.OutputStream

/** This class is used to store the metrics for a single frame
*/
final class FrameInfo(val frameIndex: Long):
final class FrameInfo(val frameIndex: Long, val metadata: Map[String, ByteString]):
var frameCount: Long = 1
var optionCount: Long = 0
var nameCount: Long = 0
Expand Down Expand Up @@ -107,13 +108,32 @@ object MetricsPrinter:
private def formatStatsIndex(
frame: FrameInfo,
): YamlMap =
YamlMap(Seq(("frame_index", YamlLong(frame.frameIndex))) ++ formatStats(frame)*)
YamlMap(
Seq(("frame_index", YamlLong(frame.frameIndex))) ++
formatMetadata(frame.metadata).map(("metadata", _)) ++
formatStats(frame)*,
)

private def formatStatsCount(
frame: FrameInfo,
): YamlMap =
// Not printing metadata in this case, as there is no upper bound on the number of frames
// and thus on the size of the collected metadata.
YamlMap(Seq(("frame_count", YamlLong(frame.frameCount))) ++ formatStats(frame)*)

private def formatMetadata(
metadata: Map[String, ByteString],
): Option[YamlMap] =
if metadata.isEmpty then None
else
Some(
YamlMap(
metadata.map { case (k, v) =>
k -> YamlString(v.toByteArray.map("%02x" format _).mkString)
}.toSeq*,
),
)

private def formatStats(
frame: FrameInfo,
): Seq[(String, YamlValue)] =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ class YamlDocBuilder(var currIndent: Int = 0):
sb.append(System.lineSeparator())
sb.append(" " * (indent + 1))
this.build(e, indent + 1)
sb.append(System.lineSeparator())
// If we are at the root level, add a blank line for readability
if indent == 0 then sb.append(System.lineSeparator())
else this.build(e, indent + 1)
if ix != v.size - 1 then sb.append(System.lineSeparator())
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,17 @@
package eu.neverblink.jelly.cli.command.rdf

import com.google.protobuf.ByteString
import eu.neverblink.jelly.cli.{ExitException, InvalidJellyFile}
import eu.neverblink.jelly.cli.command.helpers.TestFixtureHelper
import eu.ostrzyciel.jelly.core.JellyOptions
import eu.ostrzyciel.jelly.core.proto.v1.{RdfStreamFrame, RdfStreamRow}

import scala.jdk.CollectionConverters.*
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec
import org.yaml.snakeyaml.Yaml

import java.io.ByteArrayInputStream
import java.util

class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper:
Expand All @@ -25,6 +29,7 @@ class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper:
val frames = parsed.get("frames").asInstanceOf[java.util.LinkedHashMap[String, Any]]
frames.get("triple_count") should be(testCardinality)
}

"be able to return all frames separately as a valid Yaml" in withFullJellyFile(
testCode = { j =>
val (out, err) = RdfInspect.runTestCommand(List("rdf", "inspect", "--per-frame", j))
Expand All @@ -39,6 +44,7 @@ class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper:
},
frameSize = 15,
)

"handle properly separate frame metrics for a singular frame" in withFullJellyFile { j =>
val (out, err) = RdfInspect.runTestCommand(List("rdf", "inspect", "--per-frame", j))
val yaml = new Yaml()
Expand All @@ -50,6 +56,7 @@ class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper:
frames.length should be(1)
frames.map(_.get("triple_count")).sum should be(testCardinality)
}

"handle properly frame count when aggregating multiple frames" in withFullJellyFile(
testCode = { j =>
val (out, err) = RdfInspect.runTestCommand(List("rdf", "inspect", j))
Expand All @@ -65,6 +72,25 @@ class RdfInspectSpec extends AnyWordSpec with Matchers with TestFixtureHelper:
},
frameSize = 15,
)

"print frame metadata in --per-frame" in {
val inFrame = RdfStreamFrame(
rows = Seq(RdfStreamRow(JellyOptions.bigGeneralized)),
metadata = Map("key" -> ByteString.fromHex("1337ff")),
)
val inBytes = inFrame.toByteArray
RdfInspect.setStdIn(ByteArrayInputStream(inBytes))
val (out, err) = RdfInspect.runTestCommand(List("rdf", "inspect", "--per-frame"))
val yaml = new Yaml()
val parsed = yaml.load(out).asInstanceOf[java.util.Map[String, Any]]
val frame0 =
parsed.get("frames").asInstanceOf[util.ArrayList[util.HashMap[String, Any]]].get(0)
frame0.get("frame_index") should be(0)
frame0.get("metadata") should not be None
val metadata = frame0.get("metadata").asInstanceOf[util.HashMap[String, String]]
metadata.get("key") should be("1337ff")
}

"throw an error if the input file is not a valid Jelly file" in withEmptyJellyFile { j =>
val exception = intercept[ExitException] {
RdfInspect.runTestCommand(List("rdf", "inspect", j, "--debug"))
Expand Down