Get field list

datafusion/catalog/src/table.rs

@@ -18,6 +18,7 @@
 use std::any::Any;
 use std::borrow::Cow;
 use std::fmt::Debug;
+use std::collections::HashMap;
 use std::sync::Arc;
 
 use crate::session::Session;
@@ -171,6 +172,28 @@ pub trait TableProvider: Debug + Sync + Send {
         limit: Option<usize>,
     ) -> Result<Arc<dyn ExecutionPlan>>;
 
+    /// Create an [`ExecutionPlan`] with an extra parameter
+    /// specifying the deep column projections
+    /// # Deep column projection
+    ///
+    /// If specified, a datasource such as Parquet can do deep projection pushdown.
+    /// In the case of deeply nested schemas (lists in structs etc), the
+    /// implementation can return a smaller schema that rewrites the entire file
+    /// schema to return only the necessary fields, no matter where they are (top-level
+    /// or deep)
+    ///
+    async fn scan_deep(
+           &self,
+            state: &dyn Session,
+            projection: Option<&Vec<usize>>,
+            _projection_deep: Option<&HashMap<usize, Vec<String>>>,
+            filters: &[Expr],
+            limit: Option<usize>,
+        ) -> Result<Arc<dyn ExecutionPlan>> {
+            self.scan(state, projection, filters, limit).await
+    }
+
+
     /// Specify if DataFusion should provide filter expressions to the
     /// TableProvider to apply *during* the scan.
     ///

datafusion/common/Cargo.toml

@@ -68,6 +68,9 @@ recursive = { workspace = true, optional = true }
 sqlparser = { workspace = true }
 tokio = { workspace = true }
 
+prost = "0.13"
+substrait = { version = "0.53", features = ["serde"] }
+
 [target.'cfg(target_family = "wasm")'.dependencies]
 web-time = "1.1.0"
 

datafusion/common/src/config.rs

@@ -686,6 +686,9 @@ config_namespace! {
         /// then the output will be coerced to a non-view.
         /// Coerces `Utf8View` to `LargeUtf8`, and `BinaryView` to `LargeBinary`.
         pub expand_views_at_output: bool, default = false
+
+        /// disable deep column pruning
+        pub deep_column_pruning_flags:usize, default = 7
     }
 }
 

datafusion/common/src/lib.rs

@@ -56,6 +56,8 @@ pub mod types;
 pub mod utils;
 pub mod deep;
 
+pub mod substrait_tree;
+
 /// Reexport arrow crate
 pub use arrow;
 pub use column::Column;