Basic Schema Field Pruning

include/simfil/environment.h

@@ -22,6 +22,7 @@ namespace simfil
 class Expr;
 class Function;
 class Diagnostics;
+class Schema;
 struct ResultFn;
 struct Debug;
 
@@ -61,6 +62,8 @@ struct Trace
 struct Environment
 {
 public:
+    using QuerySchemaCallback = std::function<const Schema*(SchemaId)>;
+
     /**
      * Construct a SIMFIL execution environment with a string cache,
      * which is used to map field names to short integer IDs.
@@ -116,6 +119,12 @@ struct Environment
     [[nodiscard]]
     auto strings() const -> std::shared_ptr<StringPool>;
 
+    /**
+     * Query an object schema by its schema id.
+     * Returns nullptr if no callback is configured or the schema is unknown.
+     */
+    auto querySchema(SchemaId schemaId) const -> const Schema*;
+
 public:
     std::unique_ptr<std::mutex> warnMtx;
     std::vector<std::pair<std::string, std::string>> warnings;
@@ -129,6 +138,7 @@ struct Environment
     /* constant ident -> value */
     std::map<std::string, Value, CaseInsensitiveCompare> constants;
 
+    QuerySchemaCallback querySchemaCallback;
     Debug* debug = nullptr;
     std::shared_ptr<StringPool> stringPool;
 };

include/simfil/expression-visitor.h

@@ -21,6 +21,7 @@ class UnpackExpr;
 class UnaryWordOpExpr;
 class BinaryWordOpExpr;
 class FieldExpr;
+class WildcardFieldExpr;
 class PathExpr;
 class AndExpr;
 class OrExpr;
@@ -54,6 +55,7 @@ class ExprVisitor
     virtual void visit(const CallExpression& expr);
     virtual void visit(const PathExpr& expr);
     virtual void visit(const FieldExpr& expr);
+    virtual void visit(const WildcardFieldExpr& expr);
     virtual void visit(const UnpackExpr& expr);
     virtual void visit(const UnaryWordOpExpr& expr);
     virtual void visit(const BinaryWordOpExpr& expr);

include/simfil/expression.h

@@ -8,12 +8,16 @@
 #include "simfil/result.h"
 
 #include <memory>
+#include <stdexcept>
 
 namespace simfil
 {
 
+class Expr;
 class ExprVisitor;
 
+using ExprPtr = std::unique_ptr<Expr>;
+
 class Expr
 {
     friend class AST;
@@ -31,17 +35,16 @@
         VALUE,
     };
 
-    Expr() = delete;
-    explicit Expr(ExprId id)
-        : id_(id)
-    {}
-    explicit Expr(ExprId id, const Token& token)
-        : id_(id)
+    Expr() = default;
+    explicit Expr(const Token& token)
     {
         assert(token.end >= token.begin);
         sourceLocation_.offset = token.begin;
         sourceLocation_.size = token.end - token.begin;
     }
+    explicit Expr(SourceLocation location)
+        : sourceLocation_(location)
+    {}
 
     virtual ~Expr() = default;
 
@@ -56,6 +59,28 @@
         return false;
     }
 
+    /* Accept expression visitor */
+    virtual auto accept(ExprVisitor& v) const -> void = 0;
+
+    /* Get the number of child expressions */
+    virtual auto numChildren() const -> std::size_t
+    {
+        return 0;
+    }
+
+    /* Get the n-th child expression */
+    virtual auto childAt(std::size_t index) -> ExprPtr&
+    {
+        if (numChildren() == 0)
+            throw std::out_of_range("AST Child index out of range");
+        throw std::runtime_error("Missing childAt function implementation");
+    }
+
+    virtual auto childAt(std::size_t index) const -> const ExprPtr&
+    {
+        return const_cast<Expr&>(*this).childAt(index);
+    }
+
     /* Debug */
     virtual auto toString() const -> std::string = 0;
 
@@ -90,11 +115,7 @@
         return ieval(ctx, std::move(val), res);
     }
 
-    /* Accept expression visitor */
-    virtual auto accept(ExprVisitor& v) const -> void = 0;
-
     /* Source location the expression got parsed from */
-    [[nodiscard]]
     auto sourceLocation() const -> SourceLocation
     {
         return sourceLocation_;
@@ -110,12 +131,10 @@
         return ieval(ctx, value, result);
     }
 
-    ExprId id_;
+    ExprId id_ = 0;
     SourceLocation sourceLocation_;
 };
 
-using ExprPtr = std::unique_ptr<Expr>;
-
 class AST
 {
 public:
@@ -126,6 +145,8 @@
 
     ~AST();
 
+    auto reenumerate() -> void;
+
     auto expr() const -> const Expr&
     {
         return *expr_;
@@ -137,6 +158,8 @@
     }
 
 private:
+    static auto reenumerate(Expr& expr, Expr::ExprId& nextId) -> void;
+
     /* The original query string of the AST */
     std::string queryString_;
 

include/simfil/model/model.h

@@ -2,6 +2,7 @@
 #pragma once
 
 #include "simfil/model/string-pool.h"
+#include "simfil/model/schema.h"
 #include "simfil/byte-array.h"
 #include "tl/expected.hpp"
 #if defined(SIMFIL_WITH_MODEL_JSON)
@@ -274,7 +275,9 @@ class ModelPool : public Model
         size_t stringDataBytes = 0;
         size_t stringRangeBytes = 0;
         size_t objectMemberBytes = 0;
+        size_t objectSchemaBytes = 0;
         size_t arrayMemberBytes = 0;
+        size_t arraySchemaBytes = 0;
 
         [[nodiscard]] size_t totalBytes() const
         {
@@ -284,7 +287,9 @@ class ModelPool : public Model
                 + stringDataBytes
                 + stringRangeBytes
                 + objectMemberBytes
-                + arrayMemberBytes;
+                + objectSchemaBytes
+                + arrayMemberBytes
+                + arraySchemaBytes;
         }
     };
 
@@ -299,12 +304,18 @@ class ModelPool : public Model
     struct Impl;
     std::unique_ptr<Impl> impl_;
 
+    [[nodiscard]] SchemaId objectSchemaId(ArrayIndex members) const;
+    auto setObjectSchemaId(ArrayIndex members, SchemaId schemaId) -> tl::expected<void, Error>;
+    [[nodiscard]] SchemaId arraySchemaId(ArrayIndex members) const;
+    auto setArraySchemaId(ArrayIndex members, SchemaId schemaId) -> tl::expected<void, Error>;
+
     /**
      * Protected object/array member storage access,
      * so derived ModelPools can create Object/Array-derived nodes.
      */
     Object::Storage& objectMemberStorage();
     [[nodiscard]] Object::Storage const& objectMemberStorage() const;
+
     Array::Storage& arrayMemberStorage();
     [[nodiscard]] Array::Storage const& arrayMemberStorage() const;
 };

include/simfil/model/nodes.h

@@ -8,6 +8,7 @@
 #include <utility>
 
 #include "arena.h"
+#include "schema.h"
 #include "string-pool.h"
 #include "simfil/byte-array.h"
 #include "simfil/error.h"
@@ -56,8 +57,9 @@ enum class ValueType
     Bytes,
     TransientObject,
     Object,
-    Array
-    // If you add types, update TypeFlags::flags bit size!
+    Array,
+    // End
+    LAST_
 };
 
 using ScalarValueType = std::variant<
@@ -276,6 +278,9 @@ struct ModelNode
     /// Get an Object model's field names
     [[nodiscard]] virtual StringId keyAt(int64_t i) const;
 
+    /// Get the schema id for schema-aware container nodes, or NoSchemaId otherwise.
+    [[nodiscard]] virtual SchemaId schema() const;
+
     /// Get the number of children
     [[nodiscard]] virtual uint32_t size() const;
 
@@ -428,6 +433,7 @@ struct ModelNodeBase : public ModelNode
     [[nodiscard]] ModelNode::Ptr get(const StringId&) const override;
     [[nodiscard]] ModelNode::Ptr at(int64_t) const override;
     [[nodiscard]] StringId keyAt(int64_t) const override;
+    [[nodiscard]] SchemaId schema() const override;
     [[nodiscard]] uint32_t size() const override;
     bool iterate(IterCallback const&) const override {return true;}  // NOLINT (allow discard)
 
@@ -544,6 +550,9 @@ struct BaseArray : public MandatoryDerivedModelNodeBase<ModelType>
 
     bool forEach(std::function<bool(ModelNodeType const&)> const& callback) const;
 
+    [[nodiscard]] SchemaId schema() const override;
+    auto setSchema(SchemaId schemaId) -> tl::expected<void, Error>;
+
     [[nodiscard]] ValueType type() const override;
     [[nodiscard]] ModelNode::Ptr at(int64_t) const override;
     [[nodiscard]] uint32_t size() const override;
@@ -607,6 +616,9 @@ struct BaseObject : public MandatoryDerivedModelNodeBase<ModelType>
         return addFieldInternal(name, static_cast<ModelNode::Ptr>(value));
     }
 
+    [[nodiscard]] SchemaId schema() const override;
+    auto setSchema(SchemaId schemaId) -> tl::expected<void, Error>;
+
     [[nodiscard]] ValueType type() const override;
     [[nodiscard]] ModelNode::Ptr at(int64_t) const override;
     [[nodiscard]] uint32_t size() const override;

include/simfil/model/nodes.impl.h

@@ -21,6 +21,18 @@ ValueType BaseArray<ModelType, ModelNodeType>::type() const
     return ValueType::Array;
 }
 
+template <class ModelType, class ModelNodeType>
+SchemaId BaseArray<ModelType, ModelNodeType>::schema() const
+{
+    return model().arraySchemaId(members_);
+}
+
+template <class ModelType, class ModelNodeType>
+auto BaseArray<ModelType, ModelNodeType>::setSchema(SchemaId schemaId) -> tl::expected<void, Error>
+{
+    return model().setArraySchemaId(members_, schemaId);
+}
+
 template <class ModelType, class ModelNodeType>
 ModelNode::Ptr BaseArray<ModelType, ModelNodeType>::at(int64_t i) const
 {
@@ -96,6 +108,18 @@ ValueType BaseObject<ModelType, ModelNodeType>::type() const
     return ValueType::Object;
 }
 
+template <class ModelType, class ModelNodeType>
+SchemaId BaseObject<ModelType, ModelNodeType>::schema() const
+{
+    return model().objectSchemaId(members_);
+}
+
+template <class ModelType, class ModelNodeType>
+auto BaseObject<ModelType, ModelNodeType>::setSchema(SchemaId schemaId) -> tl::expected<void, Error>
+{
+    return model().setObjectSchemaId(members_, schemaId);
+}
+
 template <class ModelType, class ModelNodeType>
 ModelNode::Ptr BaseObject<ModelType, ModelNodeType>::at(int64_t i) const
 {