From 2deb06a7dfb6aacbdbee0e2e0b1c4277c1f2d492 Mon Sep 17 00:00:00 2001 From: quietvoid Date: Mon, 6 Apr 2026 18:13:26 -0400 Subject: [PATCH] export: add extension metadata levels json/csv exports Closes #374 --- Cargo.lock | 28 +++ Cargo.toml | 6 +- .../src/rpu/extension_metadata/blocks/mod.rs | 21 ++- src/commands/export.rs | 165 +++++++++++++++--- src/commands/mod.rs | 2 +- src/dovi/exporter.rs | 159 +++++++++++++++-- 6 files changed, 342 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 938ddc5..c4e6f3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -387,6 +387,27 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + [[package]] name = "darling" version = "0.20.11" @@ -528,6 +549,7 @@ dependencies = [ "bitvec_helpers", "clap", "clap_lex", + "csv", "dolby_vision", "hdr10plus", "hevc_parser", @@ -1291,6 +1313,12 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "same-file" version = "1.0.6" diff --git a/Cargo.toml b/Cargo.toml index c6eea2c..ce07f43 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,12 +22,12 @@ hdr10plus = { version = "2.1.5", features = ["json"] } anyhow = "1.0.102" clap = { version = "4.6.0", features = ["derive", "wrap_help", "deprecated"] } clap_lex = "*" +csv = "1.4.0" indicatif = "0.18.4" -serde = { version = "1.0.228", features = ["derive"] } -serde_json = { version = "1.0.149", features = ["preserve_order"] } itertools = "0.14.0" plotters = { version = "0.3.7", default-features = false, features = ["bitmap_backend", "bitmap_encoder", "all_series"] } - +serde = { version = "1.0.228", features = ["derive"] } +serde_json = { version = "1.0.149", features = ["preserve_order"] } [dev-dependencies] assert_cmd = "2.2.0" assert_fs = "1.1.3" diff --git a/dolby_vision/src/rpu/extension_metadata/blocks/mod.rs b/dolby_vision/src/rpu/extension_metadata/blocks/mod.rs index f51a0ea..c8abc01 100644 --- a/dolby_vision/src/rpu/extension_metadata/blocks/mod.rs +++ b/dolby_vision/src/rpu/extension_metadata/blocks/mod.rs @@ -4,7 +4,7 @@ use bitvec_helpers::{ }; #[cfg(feature = "serde")] -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Serialize, Serializer}; pub mod level1; pub mod level10; @@ -221,4 +221,23 @@ impl ExtMetadataBlock { Ok(()) } + + #[cfg(feature = "serde")] + pub fn serialize_inner(&self, serializer: S) -> Result { + match self { + ExtMetadataBlock::Level1(b) => b.serialize(serializer), + ExtMetadataBlock::Level2(b) => b.serialize(serializer), + ExtMetadataBlock::Level3(b) => b.serialize(serializer), + ExtMetadataBlock::Level4(b) => b.serialize(serializer), + ExtMetadataBlock::Level5(b) => b.serialize(serializer), + ExtMetadataBlock::Level6(b) => b.serialize(serializer), + ExtMetadataBlock::Level8(b) => b.serialize(serializer), + ExtMetadataBlock::Level9(b) => b.serialize(serializer), + ExtMetadataBlock::Level10(b) => b.serialize(serializer), + ExtMetadataBlock::Level11(b) => b.serialize(serializer), + ExtMetadataBlock::Level254(b) => b.serialize(serializer), + ExtMetadataBlock::Level255(b) => b.serialize(serializer), + ExtMetadataBlock::Reserved(b) => b.serialize(serializer), + } + } } diff --git a/src/commands/export.rs b/src/commands/export.rs index 72ada47..5508bea 100644 --- a/src/commands/export.rs +++ b/src/commands/export.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; use clap::{ - Args, ValueEnum, ValueHint, + Args, ValueHint, builder::{EnumValueParser, PossibleValue, TypedValueParser}, }; use clap_lex::OsStrExt as _; @@ -34,11 +34,30 @@ pub struct ExportArgs { long, short = 'd', conflicts_with = "output", - value_parser = ExportOptionParser, + value_parser = ExportDataOptionParser, value_delimiter = ',' )] pub data: Vec<(ExportData, Option)>, + #[arg( + id = "levels-format", + help = "Format to output levels exports", + long, + short = 'f', + default_value = "csv" + )] + pub levels_format: LevelsOutputFormat, + + #[arg( + id = "levels", + help = "List of key-value export parameters formatted as `key=output`, where `output` is an output file path.\nSupports multiple occurences prefixed by --levels or delimited by ','", + long, + short = 'l', + value_parser = ExportLevelsOptionParser, + value_delimiter = ',' + )] + pub levels: Vec<(ExportLevel, Option)>, + // FIXME: export single output deprecation #[arg( id = "output", @@ -62,19 +81,86 @@ pub enum ExportData { Level5, } +#[derive(clap::ValueEnum, Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum LevelsOutputFormat { + Json, + #[default] + Csv, +} + +#[derive(clap::ValueEnum, Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExportLevel { + #[value(alias = "l1")] + Level1, + #[value(alias = "l2")] + Level2, + #[value(alias = "l3")] + Level3, + #[value(alias = "l4")] + Level4, + #[value(alias = "l5")] + Level5, + #[value(alias = "l6")] + Level6, + #[value(alias = "l8")] + Level8, + #[value(alias = "l9")] + Level9, + #[value(alias = "l10")] + Level10, + #[value(alias = "l11")] + Level11, +} + impl ExportData { - pub fn default_output_file(&self) -> &'static str { + pub const fn default_output_file(&self) -> &'static str { + match self { + Self::All => "RPU_export.json", + Self::Scenes => "RPU_scenes.txt", + Self::Level5 => "RPU_L5_edit_config.json", + } + } +} + +impl ExportLevel { + pub const fn level(&self) -> u8 { + match self { + Self::Level1 => 1, + Self::Level2 => 2, + Self::Level3 => 3, + Self::Level4 => 4, + Self::Level5 => 5, + Self::Level6 => 6, + Self::Level8 => 8, + Self::Level9 => 9, + Self::Level10 => 10, + Self::Level11 => 11, + } + } + pub fn default_output_file(&self, format: LevelsOutputFormat) -> String { + let level = self.level(); + let ext = format.ext(); + + format!("L{level}_export.{ext}") + } +} + +impl LevelsOutputFormat { + pub const fn ext(&self) -> &'static str { match self { - ExportData::All => "RPU_export.json", - ExportData::Scenes => "RPU_scenes.txt", - ExportData::Level5 => "RPU_L5_edit_config.json", + Self::Json => "json", + Self::Csv => "csv", } } } #[derive(Clone)] -struct ExportOptionParser; -impl TypedValueParser for ExportOptionParser { +pub struct ExportDataOptionParser; + +#[derive(Clone)] +pub struct ExportLevelsOptionParser; + +impl TypedValueParser for ExportDataOptionParser { type Value = (ExportData, Option); fn parse_ref( @@ -83,23 +169,58 @@ impl TypedValueParser for ExportOptionParser { arg: Option<&clap::Arg>, value: &std::ffi::OsStr, ) -> Result { - let data_parser = EnumValueParser::::new(); - - if let Some((data_str, output_str)) = value.split_once("=") { - Ok(( - data_parser.parse_ref(cmd, arg, data_str)?, - output_str.to_str().map(str::parse).and_then(Result::ok), - )) - } else { - Ok((data_parser.parse_ref(cmd, arg, value)?, None)) - } + export_option_parse_ref(cmd, arg, value) } fn possible_values(&self) -> Option + '_>> { - Some(Box::new( - ExportData::value_variants() - .iter() - .filter_map(|v| v.to_possible_value()), + export_option_possible_values::() + } +} + +impl TypedValueParser for ExportLevelsOptionParser { + type Value = (ExportLevel, Option); + + fn parse_ref( + &self, + cmd: &clap::Command, + arg: Option<&clap::Arg>, + value: &std::ffi::OsStr, + ) -> Result { + export_option_parse_ref(cmd, arg, value) + } + + fn possible_values(&self) -> Option + '_>> { + export_option_possible_values::() + } +} + +fn export_option_parse_ref( + cmd: &clap::Command, + arg: Option<&clap::Arg>, + value: &std::ffi::OsStr, +) -> Result<(E, Option), clap::Error> +where + E: clap::ValueEnum + Clone + Send + Sync + 'static, +{ + let data_parser = EnumValueParser::::new(); + + if let Some((data_str, output_str)) = value.split_once("=") { + Ok(( + data_parser.parse_ref(cmd, arg, data_str)?, + output_str.to_str().map(str::parse).and_then(Result::ok), )) + } else { + Ok((data_parser.parse_ref(cmd, arg, value)?, None)) } } + +fn export_option_possible_values() -> Option>> +where + E: clap::ValueEnum + Clone + Send + Sync + 'static, +{ + Some(Box::new( + E::value_variants() + .iter() + .filter_map(|v| v.to_possible_value()), + )) +} diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 26484a1..5e9c389 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -17,7 +17,7 @@ mod remove; pub use convert::ConvertArgs; pub use demux::DemuxArgs; pub use editor::EditorArgs; -pub use export::{ExportArgs, ExportData}; +pub use export::{ExportArgs, ExportData, ExportLevel, LevelsOutputFormat}; pub use extract_rpu::ExtractRpuArgs; pub use generate::GenerateArgs; pub use info::InfoArgs; diff --git a/src/dovi/exporter.rs b/src/dovi/exporter.rs index 9dcb1d5..b0acded 100644 --- a/src/dovi/exporter.rs +++ b/src/dovi/exporter.rs @@ -5,15 +5,16 @@ use std::ops::Range; use std::path::PathBuf; use anyhow::Result; +use csv::WriterBuilder; use dolby_vision::rpu::extension_metadata::blocks::{ExtMetadataBlock, ExtMetadataBlockLevel5}; use itertools::Itertools; -use serde::Serializer; -use serde::ser::SerializeSeq; +use serde::ser::Error; +use serde::{Serialize, Serializer}; +use serde_json::json; use dolby_vision::rpu::utils::parse_rpu_file; -use serde_json::json; -use crate::commands::{ExportArgs, ExportData}; +use crate::commands::{ExportArgs, ExportData, ExportLevel, LevelsOutputFormat}; use crate::dovi::input_from_either; use super::DoviRpu; @@ -21,6 +22,27 @@ use super::DoviRpu; pub struct Exporter { input: PathBuf, data: Vec<(ExportData, Option)>, + levels_format: LevelsOutputFormat, + levels: Vec<(ExportLevel, Option)>, +} + +#[derive(Serialize)] + +struct JsonRecord<'a> { + frame: usize, + #[serde(flatten, serialize_with = "serialize_level_block")] + block: &'a ExtMetadataBlock, +} + +struct CsvHeaders<'a> { + block: &'a ExtMetadataBlock, +} + +#[derive(Serialize)] +struct CsvRecord<'a> { + frame: usize, + #[serde(serialize_with = "serialize_level_block")] + block: &'a ExtMetadataBlock, } impl Exporter { @@ -30,12 +52,19 @@ impl Exporter { input_pos, data, output, + levels_format, + levels, } = args; let input = input_from_either("editor", input, input_pos)?; - let mut exporter = Exporter { input, data }; + let mut exporter = Exporter { + input, + data, + levels_format, + levels, + }; - if exporter.data.is_empty() { + if exporter.data.is_empty() && exporter.levels.is_empty() { exporter.data.push((ExportData::All, output)); } @@ -53,6 +82,18 @@ impl Exporter { } fn execute(&self, rpus: &[DoviRpu]) -> Result<()> { + if !self.data.is_empty() { + self.export_data(rpus)?; + } + + if !self.levels.is_empty() { + self.export_levels(rpus)?; + } + + Ok(()) + } + + fn export_data(&self, rpus: &[DoviRpu]) -> Result<()> { for (data, maybe_output) in &self.data { let out_path = if let Some(out_path) = maybe_output { Cow::Borrowed(out_path) @@ -75,12 +116,7 @@ impl Exporter { println!("Exporting serialized RPU list..."); let mut ser = serde_json::Serializer::new(&mut writer); - let mut seq = ser.serialize_seq(Some(rpus.len()))?; - - for rpu in rpus { - seq.serialize_element(&rpu)?; - } - seq.end()?; + ser.collect_seq(rpus)?; } ExportData::Scenes => { println!("Exporting scenes list..."); @@ -181,4 +217,103 @@ impl Exporter { Ok(()) } + + fn export_levels(&self, rpus: &[DoviRpu]) -> Result<()> { + println!( + "Exporting extension metadata levels {}...", + self.levels.iter().map(|e| e.0.level()).join(", ") + ); + + for (export_level, maybe_output) in &self.levels { + let out_path = if let Some(out_path) = maybe_output { + Cow::Borrowed(out_path) + } else { + Cow::Owned(PathBuf::from( + export_level.default_output_file(self.levels_format), + )) + }; + + let writer_buf_len = 10_000; + let mut writer = BufWriter::with_capacity( + writer_buf_len, + File::create(out_path.as_path()).expect("Can't create file"), + ); + + let vdr_dm_data_list = rpus.iter().enumerate().filter_map(|(idx, rpu)| { + let blocks = rpu + .vdr_dm_data + .as_ref() + .map(|vdr_dm_data| vdr_dm_data.level_blocks_iter(export_level.level())); + + blocks.map(|list| (idx, list)) + }); + + match self.levels_format { + LevelsOutputFormat::Json => { + let mut ser = serde_json::Serializer::new(&mut writer); + ser.collect_seq(vdr_dm_data_list.flat_map(|(frame, blocks)| { + blocks.map(move |block| JsonRecord { frame, block }) + }))?; + } + LevelsOutputFormat::Csv => { + let mut writer = WriterBuilder::new().has_headers(false).from_writer(writer); + let mut headers_serialized = false; + + for (frame, blocks) in vdr_dm_data_list { + for block in blocks { + if !headers_serialized { + let value = CsvHeaders { block }; + writer.serialize(value)?; + headers_serialized = true; + } + + writer.serialize(CsvRecord { frame, block })?; + } + } + + writer.flush()?; + } + } + } + + Ok(()) + } +} + +fn serialize_level_block( + block: &ExtMetadataBlock, + serializer: S, +) -> Result { + block.serialize_inner(serializer) +} + +impl<'a> Serialize for CsvHeaders<'a> { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: Serializer, + { + let value = match self.block { + ExtMetadataBlock::Level1(b) => json!(b), + ExtMetadataBlock::Level2(b) => json!(b), + ExtMetadataBlock::Level3(b) => json!(b), + ExtMetadataBlock::Level4(b) => json!(b), + ExtMetadataBlock::Level5(b) => json!(b), + ExtMetadataBlock::Level6(b) => json!(b), + ExtMetadataBlock::Level8(b) => json!(b), + ExtMetadataBlock::Level9(b) => json!(b), + ExtMetadataBlock::Level10(b) => json!(b), + ExtMetadataBlock::Level11(b) => json!(b), + ExtMetadataBlock::Level254(b) => json!(b), + ExtMetadataBlock::Level255(b) => json!(b), + ExtMetadataBlock::Reserved(b) => json!(b), + }; + + value + .as_object() + .ok_or_else(|| S::Error::custom("Failed serializing headers")) + .and_then(|value| { + let headers = std::iter::once("frame").chain(value.keys().map(String::as_str)); + serializer.collect_seq(headers) + }) + } }