From 91b0a1fe029983ae6c5ba8e0c1351f988a77d50f Mon Sep 17 00:00:00 2001 From: dude Date: Tue, 20 Oct 2020 14:23:53 +0300 Subject: [PATCH 1/9] Write logic for file data resolving --- lib/qonfig.rb | 14 +++++ lib/qonfig/file_data_resolving.rb | 8 +++ lib/qonfig/file_data_resolving/mixin.rb | 24 +++++++++ lib/qonfig/file_data_resolving/resolver.rb | 53 +++++++++++++++++++ lib/qonfig/loaders/basic.rb | 7 +-- lib/qonfig/plugins/vault.rb | 19 +++++++ lib/qonfig/plugins/vault/loaders/vault.rb | 4 +- lib/qonfig/version.rb | 2 +- .../plugins/vault/load_from_yaml_spec.rb | 38 +++++++++++++ 9 files changed, 164 insertions(+), 5 deletions(-) create mode 100644 lib/qonfig/file_data_resolving.rb create mode 100644 lib/qonfig/file_data_resolving/mixin.rb create mode 100644 lib/qonfig/file_data_resolving/resolver.rb create mode 100644 spec/features/plugins/vault/load_from_yaml_spec.rb diff --git a/lib/qonfig.rb b/lib/qonfig.rb index 81ad9927..5de950de 100644 --- a/lib/qonfig.rb +++ b/lib/qonfig.rb @@ -22,6 +22,7 @@ module Qonfig require_relative 'qonfig/imports' require_relative 'qonfig/plugins' require_relative 'qonfig/compacted' + require_relative 'qonfig/file_data_resolving' # @api public # @since 0.4.0 @@ -31,6 +32,10 @@ module Qonfig # @since 0.20.0 extend Validation::PredefinitionMixin + # @api public + # @since 0.25.1 + extend FileDataResolving::Mixin + # @since 0.20.0 define_validator(:integer) { |value| value.is_a?(Integer) } # @since 0.20.0 @@ -60,6 +65,15 @@ module Qonfig # @since 0.20.0 define_validator(:not_nil) { |value| !value.nil? } + # @since 0.25.1 + define_resolver(:file) do |file_path| + ::File.read(file_path) + rescue Errno::ENOENT => error + raise Qonfig::FileNotFoundError, error.message + end + # @since 0.25.1 + set_default_resolver :file + # @since 0.12.0 register_plugin('toml', Qonfig::Plugins::TOML) # @since 0.19.0 diff --git a/lib/qonfig/file_data_resolving.rb b/lib/qonfig/file_data_resolving.rb new file mode 100644 index 00000000..a4729a59 --- /dev/null +++ b/lib/qonfig/file_data_resolving.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +# @api private +# @since 0.25.1 +module Qonfig::FileDataResolving + require_relative 'file_data_resolving/resolver' + require_relative 'file_data_resolving/mixin' +end diff --git a/lib/qonfig/file_data_resolving/mixin.rb b/lib/qonfig/file_data_resolving/mixin.rb new file mode 100644 index 00000000..7f58ba04 --- /dev/null +++ b/lib/qonfig/file_data_resolving/mixin.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +# @api public +# @since 0.25.1 +module Qonfig::FileDataResolving::Mixin + # @param scheme_name [Symbol,String] + # @param block [Block] + # @return [void] + # + # @api public + # @since 0.25.1 + def define_resolver(scheme_name, &block) + Qonfig::FileDataResolving::Resolver.add_resolver!(scheme_name, block) + end + + # @param scheme_name [Symbol,String] + # @return [void] + # + # @api public + # @since 0.25.1 + def set_default_resolver(scheme_name) + Qonfig::FileDataResolving::Resolver.set_default_resolver!(scheme_name) + end +end diff --git a/lib/qonfig/file_data_resolving/resolver.rb b/lib/qonfig/file_data_resolving/resolver.rb new file mode 100644 index 00000000..1cdfd52c --- /dev/null +++ b/lib/qonfig/file_data_resolving/resolver.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +# @api private +# @since 0.25.1 +class Qonfig::FileDataResolving::Resolver + class << self + # @param scheme [Symbol,String] + # @param resolver_proc [Proc] + # @return [void] + # + # @api private + # @since 0.25.1 + def add_resolver!(scheme, resolver_proc) + self.resolvers ||= {} + resolvers[scheme.to_sym] = resolver_proc + end + + # @param scheme_name [Symbol,String] + # @return [void] + # + # @api private + # @since 0.25.1 + def set_default_resolver!(scheme_name) + self.default_resolver = resolvers[scheme_name.to_sym] + end + + # @param file_path [String,Pathname] + # @return [String] + # @raise [Qonfig::FileNotFoundError] + # + # @api private + # @since 0.25.1 + def resolve!(file_path) + scheme_name = URI(file_path.to_s).scheme&.to_sym + resolver = resolvers[scheme_name] || default_resolver + resolver.call(file_path.to_s.split('://').last) + end + + private + + # @return [Array] + # + # @api private + # @since 0.25.1 + attr_accessor :resolvers + + # @return [Proc] + # + # @api private + # @since 0.25.1 + attr_accessor :default_resolver + end +end diff --git a/lib/qonfig/loaders/basic.rb b/lib/qonfig/loaders/basic.rb index 373c384c..404a463c 100644 --- a/lib/qonfig/loaders/basic.rb +++ b/lib/qonfig/loaders/basic.rb @@ -30,9 +30,10 @@ def load_empty_data # @api private # @since 0.5.0 def load_file(file_path, fail_on_unexist: true) - load(::File.read(file_path)) - rescue Errno::ENOENT => error - fail_on_unexist ? (raise Qonfig::FileNotFoundError, error.message) : load_empty_data + data = Qonfig::FileDataResolving::Resolver.resolve!(file_path) + load(data) + rescue Qonfig::FileNotFoundError + fail_on_unexist ? raise : load_empty_data end end end diff --git a/lib/qonfig/plugins/vault.rb b/lib/qonfig/plugins/vault.rb index a1e88b5e..4c14b2b1 100644 --- a/lib/qonfig/plugins/vault.rb +++ b/lib/qonfig/plugins/vault.rb @@ -19,6 +19,25 @@ def install! require_relative 'vault/commands/definition/load_from_vault' require_relative 'vault/commands/definition/expose_vault' require_relative 'vault/dsl' + + define_resolvers! + end + + private + + # @return [void] + # + # @since 0.25.1 + # @api private + def define_resolvers! + ::Qonfig.define_resolver(:vault) do |file_path| + *vault_path, file_name = file_path.split('/') + result = Qonfig::Loaders::Vault.load_file(vault_path.join('/'))[file_name.to_sym] + if result == nil + raise Qonfig::FileNotFoundError, "Can't load file with name #{file_name}" + end + result + end end end end diff --git a/lib/qonfig/plugins/vault/loaders/vault.rb b/lib/qonfig/plugins/vault/loaders/vault.rb index a43b7278..82b7de35 100644 --- a/lib/qonfig/plugins/vault/loaders/vault.rb +++ b/lib/qonfig/plugins/vault/loaders/vault.rb @@ -20,12 +20,14 @@ class << self # # @api private # @since 0.25.0 - def load_file(path, fail_on_unexist: true) + def load_file(path, fail_on_unexist: true, transform_values: true) data = ::Vault.with_retries(Vault::HTTPError) do ::Vault.logical.read(path.to_s)&.data&.dig(:data) end raise Qonfig::FileNotFoundError, "Path #{path} not exist" if data.nil? && fail_on_unexist result = data || empty_data + return result unless transform_values + deep_transform_values(result) rescue Vault::VaultError => error raise(Qonfig::VaultLoaderError.new(error.message).tap do |exception| diff --git a/lib/qonfig/version.rb b/lib/qonfig/version.rb index 8b565e27..1bd31573 100644 --- a/lib/qonfig/version.rb +++ b/lib/qonfig/version.rb @@ -5,5 +5,5 @@ module Qonfig # # @api public # @since 0.1.0 - VERSION = '0.25.0' + VERSION = '0.25.1' end diff --git a/spec/features/plugins/vault/load_from_yaml_spec.rb b/spec/features/plugins/vault/load_from_yaml_spec.rb new file mode 100644 index 00000000..4ecb1d8d --- /dev/null +++ b/spec/features/plugins/vault/load_from_yaml_spec.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +describe 'Plugins(vault): Load yaml from vault kv store', plugin: :vault do + before { stub_const('VaultConfig', vault_class) } + + before { allow(Vault).to receive(:logical).and_return(logical_double) } + + let(:returned_data) do + instance_double(Vault::Secret).tap do |instance| + allow(instance).to receive(:data).and_return(secret_data) + end + end + let(:logical_double) { instance_double(Vault::Logical) } + let(:secret_data) { Hash[data: { "file.yml": yaml_content }] } + let(:yaml_content) { YAML.dump(kek: "pek") } + + let(:vault_class) do + Class.new(Qonfig::DataSet) do + load_from_yaml 'vault://kv/data/development/file.yml' + end + end + + specify 'defines config object by yaml instructions' do + expect(Vault.logical).to receive(:read).with('kv/data/development').and_return(returned_data) + VaultConfig.new.settings.tap do |conf| + expect(conf).to have_attributes(kek: "pek",) + end + end + + context "when key doesn't exist" do + let(:secret_data) { Hash[data: { "other_file.yml": yaml_content }] } + + specify "raises error" do + expect(Vault.logical).to receive(:read).with('kv/data/development').and_return(returned_data) + expect { VaultConfig.new }.to raise_error(Qonfig::FileNotFoundError) + end + end +end From 1ff64f9554fc47adca92059e81817a9e39e4d8ef Mon Sep 17 00:00:00 2001 From: dude Date: Tue, 20 Oct 2020 16:39:43 +0300 Subject: [PATCH 2/9] Write documentation --- CHANGELOG.md | 17 +++++++++++++++++ README.md | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index afb1d94e..ca73ebc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,23 @@ # Changelog All notable changes to this project will be documented in this file. +## [0.25.0] - 2020-10-20 +### Added +- Added file data resolver functionality: + - Added resolver for `file` + - Added resolver for `vault` + - Added DSL methods for defining custom resolvers + +```ruby +Qonfig.define_resolver(:https) do |file_path| + Net::HTTP.get(URI("https:://#{file_path}")) +end + +class Config < Qonfig::DataSet + load_from_yaml "https://yamlhost.com/cool.yaml" +end +``` + ## [0.25.0] - 2020-09-15 ### Added - Support for **Vault** config provider: diff --git a/README.md b/README.md index a93718fa..d5139d76 100644 --- a/README.md +++ b/README.md @@ -2049,6 +2049,8 @@ end - **Daily work** - [Save to JSON file](#save-to-json-file) (`save_to_json`) - [Save to YAML file](#save-to-yaml-file) (`save_to_yaml`) +- **Define file data resolvers** + - [Working with file data resolvers](#working-with-file-data-resolvers) --- @@ -3154,6 +3156,35 @@ dynamic: 10 --- +### Working with file data resolvers + +You can define custom file data resolver + +```ruby +Qonfig.define_resolver(:http) do |file_path| + final_url = URI("http://#{file_path}") + Net::HTTP.get(final_url) +rescue SocketError => error + raise Qonfig::FileNotFoundError, error.message +end + +class AppConfig < Qonfig::DataSet + load_from_yaml "http://content-holder.com/settings.yml" +end +``` + +Also, you can set this file data resolver to default resolver. + +```ruby +Qonfig.set_default_resolver :http + +class AppConfig < Qonfig::DataSet + load_from_yaml "content-holder.com/settings.yml" # same as previous example +end +``` + +--- + ### Plugins - [toml](#plugins-toml) (provides `load_from_toml`, `save_to_toml`, `expose_toml`); @@ -3289,6 +3320,7 @@ config = Config.new - depends on `vault` gem ([link](https://github.com/hashicorp/vault-ruby)) (tested on `>= 0.1`); - provides `.load_from_vault` (works in `.load_from_yaml` manner ([doc](#load-from-yaml-file))); - provides `.expose_vault` (works in `.expose_yaml` manner ([doc](#expose-yaml))); +- provides custom file data resolver `vault://path/to/file/key.yml`; ```ruby # 1) require external dependency From f6f453ce26ae4ec0586cd5a7dc7b2d47d2182d6c Mon Sep 17 00:00:00 2001 From: dude Date: Fri, 23 Oct 2020 17:38:00 +0300 Subject: [PATCH 3/9] Fix discussions --- lib/qonfig/file_data_resolving/resolver.rb | 2 +- lib/qonfig/plugins/vault.rb | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/qonfig/file_data_resolving/resolver.rb b/lib/qonfig/file_data_resolving/resolver.rb index 1cdfd52c..d48eb16c 100644 --- a/lib/qonfig/file_data_resolving/resolver.rb +++ b/lib/qonfig/file_data_resolving/resolver.rb @@ -21,7 +21,7 @@ def add_resolver!(scheme, resolver_proc) # @api private # @since 0.25.1 def set_default_resolver!(scheme_name) - self.default_resolver = resolvers[scheme_name.to_sym] + self.default_resolver = resolvers.fetch(scheme_name.to_sym) end # @param file_path [String,Pathname] diff --git a/lib/qonfig/plugins/vault.rb b/lib/qonfig/plugins/vault.rb index 4c14b2b1..f61ffea0 100644 --- a/lib/qonfig/plugins/vault.rb +++ b/lib/qonfig/plugins/vault.rb @@ -31,8 +31,9 @@ def install! # @api private def define_resolvers! ::Qonfig.define_resolver(:vault) do |file_path| - *vault_path, file_name = file_path.split('/') - result = Qonfig::Loaders::Vault.load_file(vault_path.join('/'))[file_name.to_sym] + *vault_path, file_name = file_path.split(File::SEPARATOR) + vault_path = vault_path.join(File::SEPARATOR) + result = Qonfig::Loaders::Vault.load_file(vault_path)[file_name.to_sym] if result == nil raise Qonfig::FileNotFoundError, "Can't load file with name #{file_name}" end From 81e19cbc2d28d3e5ac8911ff58a54a3a42443598 Mon Sep 17 00:00:00 2001 From: dude Date: Thu, 5 Nov 2020 12:21:55 +0300 Subject: [PATCH 4/9] Backward compatability for old rubies --- lib/qonfig/file_data_resolving/resolver.rb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/qonfig/file_data_resolving/resolver.rb b/lib/qonfig/file_data_resolving/resolver.rb index d48eb16c..749607b9 100644 --- a/lib/qonfig/file_data_resolving/resolver.rb +++ b/lib/qonfig/file_data_resolving/resolver.rb @@ -11,8 +11,8 @@ class << self # @api private # @since 0.25.1 def add_resolver!(scheme, resolver_proc) - self.resolvers ||= {} - resolvers[scheme.to_sym] = resolver_proc + @resolvers ||= {} + @resolvers[scheme.to_sym] = resolver_proc end # @param scheme_name [Symbol,String] @@ -21,7 +21,7 @@ def add_resolver!(scheme, resolver_proc) # @api private # @since 0.25.1 def set_default_resolver!(scheme_name) - self.default_resolver = resolvers.fetch(scheme_name.to_sym) + @default_resolver = resolvers.fetch(scheme_name.to_sym) end # @param file_path [String,Pathname] @@ -31,7 +31,8 @@ def set_default_resolver!(scheme_name) # @api private # @since 0.25.1 def resolve!(file_path) - scheme_name = URI(file_path.to_s).scheme&.to_sym + scheme_name = URI(file_path.to_s).scheme + scheme_name = scheme_name.nil? ? nil : scheme_name.to_sym resolver = resolvers[scheme_name] || default_resolver resolver.call(file_path.to_s.split('://').last) end @@ -42,12 +43,12 @@ def resolve!(file_path) # # @api private # @since 0.25.1 - attr_accessor :resolvers + attr_reader :resolvers # @return [Proc] # # @api private # @since 0.25.1 - attr_accessor :default_resolver + attr_reader :default_resolver end end From b2bcd393e0eeabdbce9be6ec6c2c3aa118e43d0d Mon Sep 17 00:00:00 2001 From: dude Date: Thu, 5 Nov 2020 12:37:38 +0300 Subject: [PATCH 5/9] Make code cooler --- lib/qonfig/file_data_resolving/resolver.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/qonfig/file_data_resolving/resolver.rb b/lib/qonfig/file_data_resolving/resolver.rb index 749607b9..0e1d51d3 100644 --- a/lib/qonfig/file_data_resolving/resolver.rb +++ b/lib/qonfig/file_data_resolving/resolver.rb @@ -32,7 +32,7 @@ def set_default_resolver!(scheme_name) # @since 0.25.1 def resolve!(file_path) scheme_name = URI(file_path.to_s).scheme - scheme_name = scheme_name.nil? ? nil : scheme_name.to_sym + scheme_name = scheme_name.to_sym if scheme_name != nil resolver = resolvers[scheme_name] || default_resolver resolver.call(file_path.to_s.split('://').last) end From 40ee7f8f877500c0cbb4f274f6115a7bf5f40fac Mon Sep 17 00:00:00 2001 From: dude Date: Thu, 5 Nov 2020 12:39:30 +0300 Subject: [PATCH 6/9] Fix rubocop linting --- lib/qonfig/file_data_resolving/resolver.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/qonfig/file_data_resolving/resolver.rb b/lib/qonfig/file_data_resolving/resolver.rb index 0e1d51d3..4064284e 100644 --- a/lib/qonfig/file_data_resolving/resolver.rb +++ b/lib/qonfig/file_data_resolving/resolver.rb @@ -32,7 +32,7 @@ def set_default_resolver!(scheme_name) # @since 0.25.1 def resolve!(file_path) scheme_name = URI(file_path.to_s).scheme - scheme_name = scheme_name.to_sym if scheme_name != nil + scheme_name = scheme_name.to_sym unless scheme_name == nil resolver = resolvers[scheme_name] || default_resolver resolver.call(file_path.to_s.split('://').last) end From fa612b1a68b94ecd55ca2c49c961d5d155c66708 Mon Sep 17 00:00:00 2001 From: JustAnotherDude Date: Tue, 22 Dec 2020 01:21:24 +0300 Subject: [PATCH 7/9] Add support for kv storage, add resolve options support, small fixes --- CHANGELOG.md | 3 +- README.md | 4 +++ .../commands/definition/load_from_json.rb | 10 +++++- .../commands/definition/load_from_yaml.rb | 13 ++++++-- .../commands/instantiation/values_file.rb | 16 ++++++++-- lib/qonfig/data_set.rb | 26 +++++++++++++--- lib/qonfig/dsl.rb | 12 ++++--- lib/qonfig/file_data_resolving/resolver.rb | 4 +-- lib/qonfig/loaders/basic.rb | 4 +-- .../commands/definition/load_from_toml.rb | 12 +++++-- lib/qonfig/plugins/toml/data_set.rb | 8 +++-- lib/qonfig/plugins/toml/dsl.rb | 4 +-- lib/qonfig/plugins/vault.rb | 6 ++-- .../commands/definition/load_from_vault.rb | 13 ++++++-- lib/qonfig/plugins/vault/dsl.rb | 5 +-- lib/qonfig/plugins/vault/loaders/vault.rb | 27 +++++++++++++--- spec/features/plugins/vault/context.rb | 10 ++++++ .../plugins/vault/expose_vault_spec.rb | 8 ++--- .../plugins/vault/load_from_vault_spec.rb | 27 ++++++++++++++-- .../plugins/vault/load_from_yaml_spec.rb | 31 ++++++++++++++++--- 20 files changed, 193 insertions(+), 50 deletions(-) create mode 100644 spec/features/plugins/vault/context.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index ca73ebc5..ad003ed6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,13 @@ # Changelog All notable changes to this project will be documented in this file. -## [0.25.0] - 2020-10-20 +## [0.25.0] - 2020-12-22 ### Added - Added file data resolver functionality: - Added resolver for `file` - Added resolver for `vault` - Added DSL methods for defining custom resolvers +- Added support for kv storage for vault ```ruby Qonfig.define_resolver(:https) do |file_path| diff --git a/README.md b/README.md index d5139d76..1f83ea29 100644 --- a/README.md +++ b/README.md @@ -3321,6 +3321,7 @@ config = Config.new - provides `.load_from_vault` (works in `.load_from_yaml` manner ([doc](#load-from-yaml-file))); - provides `.expose_vault` (works in `.expose_yaml` manner ([doc](#expose-yaml))); - provides custom file data resolver `vault://path/to/file/key.yml`; +- you can use version option to use specific version of config in kv storage (dsl and resolver); ```ruby # 1) require external dependency @@ -3352,6 +3353,9 @@ Qonfig.plugin(:vault) - External validation class with an importing api for better custom validations; - Setting value changement trace (in `anyway_config` manner); - Instantiation and reloading callbacks; + - Add supported formats for resolvers: for example, if you use vault resolver, you can't use json format; + - Refactor params passed to methods: use value objects instead; + - Refactor loaders: use base class for them; ## Build diff --git a/lib/qonfig/commands/definition/load_from_json.rb b/lib/qonfig/commands/definition/load_from_json.rb index e64fff5b..93f859fa 100644 --- a/lib/qonfig/commands/definition/load_from_json.rb +++ b/lib/qonfig/commands/definition/load_from_json.rb @@ -18,14 +18,22 @@ class Qonfig::Commands::Definition::LoadFromJSON < Qonfig::Commands::Base # @sicne 0.5.0 attr_reader :strict + # @return [Hash] + # + # @api private + # @since 0.25.1 + attr_reader :file_resolve_options + # @param file_path [String, Pathname] # @option strict [Boolean] + # @option file_resolve_options [Hash] # # @api private # @since 0.5.0 - def initialize(file_path, strict: true) + def initialize(file_path, strict: true, file_resolve_options: {}) @file_path = file_path @strict = strict + @file_resolve_options = file_resolve_options end # @param data_set [Qonfig::DataSet] diff --git a/lib/qonfig/commands/definition/load_from_yaml.rb b/lib/qonfig/commands/definition/load_from_yaml.rb index a393048c..4b4c64cc 100644 --- a/lib/qonfig/commands/definition/load_from_yaml.rb +++ b/lib/qonfig/commands/definition/load_from_yaml.rb @@ -18,14 +18,22 @@ class Qonfig::Commands::Definition::LoadFromYAML < Qonfig::Commands::Base # @since 0.2.0 attr_reader :strict + # @return [Hash] + # + # @api private + # @since 0.25.1 + attr_reader :file_resolve_options + # @param file_path [String, Pathname] # @option strict [Boolean] + # @option file_resolve_options [Hash] # # @api private # @since 0.2.0 - def initialize(file_path, strict: true) + def initialize(file_path, strict: true, file_resolve_options: {}) @file_path = file_path @strict = strict + @file_resolve_options = file_resolve_options end # @param data_set [Qonfig::DataSet] @@ -37,7 +45,8 @@ def initialize(file_path, strict: true) # @api private # @since 0.2.0 def call(data_set, settings) - yaml_data = Qonfig::Loaders::YAML.load_file(file_path, fail_on_unexist: strict) + yaml_data = Qonfig::Loaders::YAML + .load_file(file_path, fail_on_unexist: strict, **file_resolve_options) raise( Qonfig::IncompatibleYAMLStructureError, diff --git a/lib/qonfig/commands/instantiation/values_file.rb b/lib/qonfig/commands/instantiation/values_file.rb index 6ac83420..599215e2 100644 --- a/lib/qonfig/commands/instantiation/values_file.rb +++ b/lib/qonfig/commands/instantiation/values_file.rb @@ -61,22 +61,30 @@ class Qonfig::Commands::Instantiation::ValuesFile < Qonfig::Commands::Base # @since 0.17.0 attr_reader :expose + # @return [Hash] + # + # @api private + # @since 0.25.1 + attr_reader :file_resolve_options + # @param file_path [String, Symbol, Pathname] # @param caller_location [String] # @option format [String, Symbol] # @option strict [Boolean] # @option expose [NilClass, String, Symbol] + # @option file_resolve_options [Hash] # @return [void] # # @api private # @since 0.17.0 - # @version 0.22.0 + # @version 0.25.1 def initialize( file_path, caller_location, format: DEFAULT_FORMAT, strict: DEFAULT_STRICT_BEHAVIOR, - expose: NO_EXPOSE + expose: NO_EXPOSE, + file_resolve_options: {} ) prevent_incompatible_attributes!(file_path, format, strict, expose) @@ -85,6 +93,7 @@ def initialize( @format = format @strict = strict @expose = expose + @file_resolve_options = file_resolve_options end # @param data_set [Qonfig::DataSet] @@ -117,7 +126,8 @@ def load_settings_values # @api private # @since 0.17.0 def load_from_file - Qonfig::Loaders.resolve(format).load_file(file_path, fail_on_unexist: strict).tap do |values| + load_args = [file_path, { fail_on_unexist: strict, **file_resolve_options }] + Qonfig::Loaders.resolve(format).load_file(*load_args).tap do |values| raise( Qonfig::IncompatibleDataStructureError, 'Setting values must be a hash-like structure' diff --git a/lib/qonfig/data_set.rb b/lib/qonfig/data_set.rb index 61313bb9..115b49b0 100644 --- a/lib/qonfig/data_set.rb +++ b/lib/qonfig/data_set.rb @@ -101,6 +101,7 @@ def reload!(settings_map = {}, &configurations) # @option format [String, Symbol] # @option strict [Boolean] # @option expose [NilClass, String, Symbol] Environment key + # @option **file_resolve_options [Hash] # @param configurations [Block] # @return [void] # @@ -109,10 +110,18 @@ def reload!(settings_map = {}, &configurations) # @api public # @since 0.17.0 # @version 0.22.0 - def load_from_file(file_path, format: :dynamic, strict: true, expose: nil, &configurations) + def load_from_file( + file_path, + format: :dynamic, + strict: true, + expose: nil, + **file_resolve_options, + &configurations + ) thread_safe_access do load_setting_values_from_file( - file_path, format: format, strict: strict, expose: expose, &configurations + file_path, format: format, strict: strict, + expose: expose, file_resolve_options: file_resolve_options, &configurations ) end end @@ -120,6 +129,7 @@ def load_from_file(file_path, format: :dynamic, strict: true, expose: nil, &conf # @param file_path [String, Symbol, Pathname] # @option strict [Boolean] # @option expose [NilClass, String, Symbol] Environment key + # @option **file_resolve_options [Hash] # @param configurations [Block] # @return [void] # @@ -128,8 +138,11 @@ def load_from_file(file_path, format: :dynamic, strict: true, expose: nil, &conf # @api public # @since 0.17.0 # @version 0.22.0 - def load_from_yaml(file_path, strict: true, expose: nil, &configurations) - load_from_file(file_path, format: :yml, strict: strict, expose: expose, &configurations) + def load_from_yaml(file_path, strict: true, expose: nil, **file_resolve_options, &configurations) + load_from_file( + file_path, format: :yml, strict: strict, + expose: expose, **file_resolve_options, &configurations + ) end # @param file_path [String, Symbol, Pathname] @@ -521,6 +534,7 @@ def load!(settings_map = {}, &configurations) # @option strict [Boolean] # @option expose [NilClass, String, Symbol] # @option callcer_location [NilClass, String] + # @option **file_resolve_options [Hash] # @param configurations [Block] # @return [void] # @@ -535,10 +549,12 @@ def load_setting_values_from_file( strict: true, expose: nil, caller_location: nil, + file_resolve_options: {}, &configurations ) Qonfig::Commands::Instantiation::ValuesFile.new( - file_path, caller_location, format: format, strict: strict, expose: expose + file_path, caller_location, format: format, + strict: strict, expose: expose, file_resolve_options: file_resolve_options ).call(self, settings) apply_settings(&configurations) end diff --git a/lib/qonfig/dsl.rb b/lib/qonfig/dsl.rb index a9432430..40eb6275 100644 --- a/lib/qonfig/dsl.rb +++ b/lib/qonfig/dsl.rb @@ -156,15 +156,16 @@ def compose(data_set_klass) # @param file_path [String, Pathname] # @option strict [Boolean] + # @option **file_resolve_options [Hash] # @return [void] # # @see Qonfig::Commands::Definition::LoadFromYAML # # @api public # @since 0.2.0 - def load_from_yaml(file_path, strict: true) + def load_from_yaml(file_path, strict: true, **file_resolve_options) definition_commands << Qonfig::Commands::Definition::LoadFromYAML.new( - file_path, strict: strict + file_path, strict: strict, file_resolve_options: file_resolve_options ) end @@ -202,14 +203,17 @@ def load_from_env(convert_values: false, prefix: nil, trim_prefix: false) # @param file_path [String, Pathname] # @option strict [Boolean] + # @option **file_resolve_options [Hash] # @return [void] # # @see Qonfig::Commands::Definition::LoadFromJSON # # @api public # @since 0.5.0 - def load_from_json(file_path, strict: true) - definition_commands << Qonfig::Commands::Definition::LoadFromJSON.new(file_path, strict: strict) + def load_from_json(file_path, strict: true, **file_resolve_options) + definition_commands << Qonfig::Commands::Definition::LoadFromJSON.new( + file_path, strict: strict, file_resolve_options: file_resolve_options + ) end # @param file_path [String, Pathname] diff --git a/lib/qonfig/file_data_resolving/resolver.rb b/lib/qonfig/file_data_resolving/resolver.rb index 4064284e..a149d1c4 100644 --- a/lib/qonfig/file_data_resolving/resolver.rb +++ b/lib/qonfig/file_data_resolving/resolver.rb @@ -30,11 +30,11 @@ def set_default_resolver!(scheme_name) # # @api private # @since 0.25.1 - def resolve!(file_path) + def resolve!(file_path, **options) scheme_name = URI(file_path.to_s).scheme scheme_name = scheme_name.to_sym unless scheme_name == nil resolver = resolvers[scheme_name] || default_resolver - resolver.call(file_path.to_s.split('://').last) + resolver.call(file_path.to_s.split('://').last, **options) end private diff --git a/lib/qonfig/loaders/basic.rb b/lib/qonfig/loaders/basic.rb index 404a463c..afb48dbb 100644 --- a/lib/qonfig/loaders/basic.rb +++ b/lib/qonfig/loaders/basic.rb @@ -29,8 +29,8 @@ def load_empty_data # # @api private # @since 0.5.0 - def load_file(file_path, fail_on_unexist: true) - data = Qonfig::FileDataResolving::Resolver.resolve!(file_path) + def load_file(file_path, fail_on_unexist: true, **options) + data = Qonfig::FileDataResolving::Resolver.resolve!(file_path, **options) load(data) rescue Qonfig::FileNotFoundError fail_on_unexist ? raise : load_empty_data diff --git a/lib/qonfig/plugins/toml/commands/definition/load_from_toml.rb b/lib/qonfig/plugins/toml/commands/definition/load_from_toml.rb index 48752b80..6c95c457 100644 --- a/lib/qonfig/plugins/toml/commands/definition/load_from_toml.rb +++ b/lib/qonfig/plugins/toml/commands/definition/load_from_toml.rb @@ -19,14 +19,21 @@ class Qonfig::Commands::Definition::LoadFromTOML < Qonfig::Commands::Base # @since 0.12.0 attr_reader :strict + # @return [Hash] + # + # @api private + # @since 0.25.1 + attr_reader :file_resolve_options + # @param file_path [String] # @option strict [Boolean] # # @api private # @since 0.12.0 - def initialize(file_path, strict: true) + def initialize(file_path, strict: true, file_resolve_options: {}) @file_path = file_path @strict = strict + @file_resolve_options = file_resolve_options end # @param data_set [Qonfig::DataSet] @@ -36,7 +43,8 @@ def initialize(file_path, strict: true) # @api private # @since 0.12.0 def call(data_set, settings) - toml_data = Qonfig::Loaders::TOML.load_file(file_path, fail_on_unexist: strict) + toml_data = Qonfig::Loaders::TOML + .load_file(file_path, fail_on_unexist: strict, **file_resolve_options) toml_based_settings = build_data_set_klass(toml_data).new.settings settings.__append_settings__(toml_based_settings) end diff --git a/lib/qonfig/plugins/toml/data_set.rb b/lib/qonfig/plugins/toml/data_set.rb index 67b6eee8..4ce94a00 100644 --- a/lib/qonfig/plugins/toml/data_set.rb +++ b/lib/qonfig/plugins/toml/data_set.rb @@ -20,6 +20,7 @@ def save_to_toml(path:, options: Qonfig::Uploaders::TOML::DEFAULT_OPTIONS, &valu # @param file_path [String, Pathmame] # @option strict [Boolean] # @option expose [NilClass, String, Symbol] Environment key + # @option **file_resolve_options [Hash] # @param configuration [Block] # @return [void] # @@ -28,7 +29,10 @@ def save_to_toml(path:, options: Qonfig::Uploaders::TOML::DEFAULT_OPTIONS, &valu # @api public # @since 0.17.0 # @version 0.21.0 - def load_from_toml(file_path, strict: true, expose: nil, &configuration) - load_from_file(file_path, format: :toml, strict: strict, expose: expose, &configuration) + def load_from_toml(file_path, strict: true, expose: nil, **file_resolve_options, &configuration) + load_from_file( + file_path, format: :toml, strict: strict, + expose: expose, **file_resolve_options, &configuration + ) end end diff --git a/lib/qonfig/plugins/toml/dsl.rb b/lib/qonfig/plugins/toml/dsl.rb index 8a3e97d4..abda57a2 100644 --- a/lib/qonfig/plugins/toml/dsl.rb +++ b/lib/qonfig/plugins/toml/dsl.rb @@ -12,9 +12,9 @@ module Qonfig::DSL # @api public # @since 0.12.0 # @version 0.20.0 - def load_from_toml(file_path, strict: true) + def load_from_toml(file_path, strict: true, **file_resolve_options) definition_commands << Qonfig::Commands::Definition::LoadFromTOML.new( - file_path, strict: strict + file_path, strict: strict, file_resolve_options: file_resolve_options ) end diff --git a/lib/qonfig/plugins/vault.rb b/lib/qonfig/plugins/vault.rb index f61ffea0..98b94d01 100644 --- a/lib/qonfig/plugins/vault.rb +++ b/lib/qonfig/plugins/vault.rb @@ -30,10 +30,12 @@ def install! # @since 0.25.1 # @api private def define_resolvers! - ::Qonfig.define_resolver(:vault) do |file_path| + ::Qonfig.define_resolver(:vault) do |file_path, **options| *vault_path, file_name = file_path.split(File::SEPARATOR) vault_path = vault_path.join(File::SEPARATOR) - result = Qonfig::Loaders::Vault.load_file(vault_path)[file_name.to_sym] + files = Qonfig::Loaders::Vault + .load_file(vault_path, **options, transform_values: false) + result = files[file_name.to_sym] if result == nil raise Qonfig::FileNotFoundError, "Can't load file with name #{file_name}" end diff --git a/lib/qonfig/plugins/vault/commands/definition/load_from_vault.rb b/lib/qonfig/plugins/vault/commands/definition/load_from_vault.rb index 7236744a..e154ae08 100644 --- a/lib/qonfig/plugins/vault/commands/definition/load_from_vault.rb +++ b/lib/qonfig/plugins/vault/commands/definition/load_from_vault.rb @@ -18,14 +18,22 @@ class Qonfig::Commands::Definition::LoadFromVault < Qonfig::Commands::Base # @since 0.25.0 attr_reader :strict + # @return [Hash] + # + # @api private + # @since 0.25.1 + attr_reader :file_resolve_options + # @param path [String] # @option strict [Boolean] + # @option file_resolve_options [Hash] # # @api private # @since 0.25.0 - def initialize(path, strict: true) + def initialize(path, strict: true, file_resolve_options: {}) @path = path @strict = strict + @file_resolve_options = file_resolve_options end # @param data_set [Qonfig::DataSet] @@ -35,7 +43,8 @@ def initialize(path, strict: true) # @api private # @since 0.25.0 def call(_data_set, settings) - vault_data = Qonfig::Loaders::Vault.load_file(path, fail_on_unexist: strict) + vault_data = Qonfig::Loaders::Vault + .load_file(path, fail_on_unexist: strict, **file_resolve_options) vault_based_settings = build_data_set_klass(vault_data).new.settings settings.__append_settings__(vault_based_settings) end diff --git a/lib/qonfig/plugins/vault/dsl.rb b/lib/qonfig/plugins/vault/dsl.rb index 010af2c9..3eca34f2 100644 --- a/lib/qonfig/plugins/vault/dsl.rb +++ b/lib/qonfig/plugins/vault/dsl.rb @@ -5,15 +5,16 @@ module Qonfig::DSL # @param path [String, Pathname] # @option strict [Boolean] + # @option **file_resolve_options [Hash] # @return [void] # # @see Qonfig::Commands::Definition::LoadFromVault # # @api public # @since 0.25.0 - def load_from_vault(path, strict: true) + def load_from_vault(path, strict: true, **file_resolve_options) definition_commands << Qonfig::Commands::Definition::LoadFromVault.new( - path, strict: strict + path, strict: strict, file_resolve_options: file_resolve_options ) end diff --git a/lib/qonfig/plugins/vault/loaders/vault.rb b/lib/qonfig/plugins/vault/loaders/vault.rb index 82b7de35..2af7a70c 100644 --- a/lib/qonfig/plugins/vault/loaders/vault.rb +++ b/lib/qonfig/plugins/vault/loaders/vault.rb @@ -20,11 +20,9 @@ class << self # # @api private # @since 0.25.0 - def load_file(path, fail_on_unexist: true, transform_values: true) - data = ::Vault.with_retries(Vault::HTTPError) do - ::Vault.logical.read(path.to_s)&.data&.dig(:data) - end - raise Qonfig::FileNotFoundError, "Path #{path} not exist" if data.nil? && fail_on_unexist + def load_file(path, fail_on_unexist: true, transform_values: true, version: nil) + data = load_data(path, version) + raise Qonfig::FileNotFoundError, "Path #{path} not exist" if data == nil && fail_on_unexist result = data || empty_data return result unless transform_values @@ -45,6 +43,25 @@ def empty_data private + # @param file_path [String] + # @param version [Integer] + # @return [Object] + # + # @api private + # @since 0.25.1 + def load_data(file_path, version) + response = ::Vault.with_retries(::Vault::HTTPError) do + if version == nil + ::Vault.logical.read(file_path.to_s) + else + mount_path, secret_path = file_path.to_s.split(::File::Separator, 2) + ::Vault.kv(mount_path).read(secret_path, version) + end + end + + response&.data&.dig(:data) + end + # @param vault_data [Hash] # @return [Object] # diff --git a/spec/features/plugins/vault/context.rb b/spec/features/plugins/vault/context.rb new file mode 100644 index 00000000..5f758998 --- /dev/null +++ b/spec/features/plugins/vault/context.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +shared_context "vault context" do + before { allow(Vault).to receive(:logical).and_return(logical_double) } + + before { allow(Vault).to receive(:kv).and_return(kv_double) } + + let(:logical_double) { instance_double(Vault::Logical) } + let(:kv_double) { instance_double(Vault::KV) } +end diff --git a/spec/features/plugins/vault/expose_vault_spec.rb b/spec/features/plugins/vault/expose_vault_spec.rb index 56b09f1e..6d995013 100644 --- a/spec/features/plugins/vault/expose_vault_spec.rb +++ b/spec/features/plugins/vault/expose_vault_spec.rb @@ -1,19 +1,17 @@ # frozen_string_literal: true +require_relative 'context' + describe 'Plugins(vault): expose vault', plugin: :vault do before { stub_const('VaultConfig', vault_class) } - before { allow(Vault).to receive(:logical).and_return(logical_double) } - - let(:logical_double) { instance_double(Vault::Logical) } - let(:returned_data) do instance_double(Vault::Secret).tap do |instance| allow(instance).to receive(:data).and_return(secret_data) end end let(:secret_data) do - { data: { production: { kek: 'pek', cheburek: true }, other_key: "<%= 1 + 1 %>" } } + { data: { production: { kek: 'pek', cheburek: true }, other_key: '<%= 1 + 1 %>' } } end let(:vault_class) do diff --git a/spec/features/plugins/vault/load_from_vault_spec.rb b/spec/features/plugins/vault/load_from_vault_spec.rb index 690751b3..510774ea 100644 --- a/spec/features/plugins/vault/load_from_vault_spec.rb +++ b/spec/features/plugins/vault/load_from_vault_spec.rb @@ -1,16 +1,17 @@ # frozen_string_literal: true +require_relative 'context' + describe 'Plugins(vault): Load from vault kv store', plugin: :vault do - before { stub_const('VaultConfig', vault_class) } + include_context 'vault context' - before { allow(Vault).to receive(:logical).and_return(logical_double) } + before { stub_const('VaultConfig', vault_class) } let(:returned_data) do instance_double(Vault::Secret).tap do |instance| allow(instance).to receive(:data).and_return(secret_data) end end - let(:logical_double) { instance_double(Vault::Logical) } let(:secret_data) { Hash[data: { kek: true, pek: 'cheburek', nested: Hash[key: 123] }] } let(:vault_class) do @@ -77,4 +78,24 @@ expect { VaultConfig.new }.to raise_error(Qonfig::VaultLoaderError, 'Cool error') end end + + context 'when version specified' do + let(:vault_class) do + Class.new(Qonfig::DataSet) do + load_from_vault 'kv/data/development', version: 2 + end + end + + let(:expected_path) { 'data/development' } + + specify 'uses kv store engine' do + expect(Vault).to receive(:kv).with('kv').and_return(kv_double) + expect(kv_double).to receive(:read).with(expected_path, 2).and_return(returned_data) + + VaultConfig.new.settings.tap do |conf| + expect(conf).to have_attributes(kek: true, pek: 'cheburek') + expect(conf.nested.key).to eq(123) + end + end + end end diff --git a/spec/features/plugins/vault/load_from_yaml_spec.rb b/spec/features/plugins/vault/load_from_yaml_spec.rb index 4ecb1d8d..3d1b72d8 100644 --- a/spec/features/plugins/vault/load_from_yaml_spec.rb +++ b/spec/features/plugins/vault/load_from_yaml_spec.rb @@ -1,16 +1,18 @@ # frozen_string_literal: true +require_relative 'context' + describe 'Plugins(vault): Load yaml from vault kv store', plugin: :vault do - before { stub_const('VaultConfig', vault_class) } + include_context 'vault context' - before { allow(Vault).to receive(:logical).and_return(logical_double) } + before { stub_const('VaultConfig', vault_class) } let(:returned_data) do instance_double(Vault::Secret).tap do |instance| allow(instance).to receive(:data).and_return(secret_data) end end - let(:logical_double) { instance_double(Vault::Logical) } + let(:secret_data) { Hash[data: { "file.yml": yaml_content }] } let(:yaml_content) { YAML.dump(kek: "pek") } @@ -23,16 +25,35 @@ specify 'defines config object by yaml instructions' do expect(Vault.logical).to receive(:read).with('kv/data/development').and_return(returned_data) VaultConfig.new.settings.tap do |conf| - expect(conf).to have_attributes(kek: "pek",) + expect(conf).to have_attributes(kek: "pek") end end context "when key doesn't exist" do let(:secret_data) { Hash[data: { "other_file.yml": yaml_content }] } - specify "raises error" do + specify 'raises error' do expect(Vault.logical).to receive(:read).with('kv/data/development').and_return(returned_data) expect { VaultConfig.new }.to raise_error(Qonfig::FileNotFoundError) end end + + context 'when version specified' do + let(:vault_class) do + Class.new(Qonfig::DataSet) do + load_from_yaml 'vault://kv/data/development/file.yml', version: 2 + end + end + + let(:expected_path) { 'data/development' } + + specify 'uses kv store engine' do + expect(Vault).to receive(:kv).with('kv').and_return(kv_double) + expect(kv_double).to receive(:read).with(expected_path, 2).and_return(returned_data) + + VaultConfig.new.settings.tap do |conf| + expect(conf).to have_attributes(kek: 'pek') + end + end + end end From a8b695a118eea15447be956e4c33f09fb5bc91f9 Mon Sep 17 00:00:00 2001 From: JustAnotherDude Date: Tue, 22 Dec 2020 01:54:00 +0300 Subject: [PATCH 8/9] Fix bugs --- CHANGELOG.md | 1 + README.md | 1 + .../vault/commands/definition/expose_vault.rb | 19 +++++++++++++------ lib/qonfig/plugins/vault/dsl.rb | 5 +++-- lib/qonfig/plugins/vault/loaders/vault.rb | 14 +++++++------- .../plugins/vault/expose_vault_spec.rb | 19 ++++++++++--------- .../plugins/vault/load_from_vault_spec.rb | 10 +++++----- .../plugins/vault/load_from_yaml_spec.rb | 10 +++++----- 8 files changed, 45 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad003ed6..075981db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file. - Added resolver for `vault` - Added DSL methods for defining custom resolvers - Added support for kv storage for vault +- Fix using of non-kv storage ```ruby Qonfig.define_resolver(:https) do |file_path| diff --git a/README.md b/README.md index 1f83ea29..9e1baa11 100644 --- a/README.md +++ b/README.md @@ -3322,6 +3322,7 @@ config = Config.new - provides `.expose_vault` (works in `.expose_yaml` manner ([doc](#expose-yaml))); - provides custom file data resolver `vault://path/to/file/key.yml`; - you can use version option to use specific version of config in kv storage (dsl and resolver); +- kv storage is used by default, but you can use logical storage by setting `use_kv` to `false`; ```ruby # 1) require external dependency diff --git a/lib/qonfig/plugins/vault/commands/definition/expose_vault.rb b/lib/qonfig/plugins/vault/commands/definition/expose_vault.rb index ec8d1b9f..1bc129bc 100644 --- a/lib/qonfig/plugins/vault/commands/definition/expose_vault.rb +++ b/lib/qonfig/plugins/vault/commands/definition/expose_vault.rb @@ -42,6 +42,12 @@ class Qonfig::Commands::Definition::ExposeVault < Qonfig::Commands::Base # @since 0.25.0 attr_reader :env + # @return [Hash] + # + # @api private + # @since 0.25.1 + attr_reader :file_resolve_options + # @param path [String Pathname] # @option strict [Boolean] # @option via [Symbol] @@ -50,7 +56,7 @@ class Qonfig::Commands::Definition::ExposeVault < Qonfig::Commands::Base # # @api private # @since 0.25.0 - def initialize(path, strict: true, via:, env:) + def initialize(path, strict: true, via:, env:, file_resolve_options: {}) unless env.is_a?(Symbol) || env.is_a?(String) || env.is_a?(Numeric) raise Qonfig::ArgumentError, ':env should be a string or a symbol' end @@ -58,10 +64,11 @@ def initialize(path, strict: true, via:, env:) raise Qonfig::ArgumentError, ':env should be provided' if env.to_s.empty? raise Qonfig::ArgumentError, 'used :via is unsupported' unless EXPOSERS.key?(via) - @path = path - @strict = strict - @via = via - @env = env + @path = path + @strict = strict + @via = via + @env = env + @file_resolve_options = file_resolve_options end # @param data_set [Qonfig::DataSet] @@ -128,7 +135,7 @@ def expose_env_key!(settings) # @api private # @since 0.25.0 def load_vault_data(path) - Qonfig::Loaders::Vault.load_file(path, fail_on_unexist: strict) + Qonfig::Loaders::Vault.load_file(path, fail_on_unexist: strict, **file_resolve_options) end # @param vault_data [Hash] diff --git a/lib/qonfig/plugins/vault/dsl.rb b/lib/qonfig/plugins/vault/dsl.rb index 3eca34f2..e0bea51f 100644 --- a/lib/qonfig/plugins/vault/dsl.rb +++ b/lib/qonfig/plugins/vault/dsl.rb @@ -22,15 +22,16 @@ def load_from_vault(path, strict: true, **file_resolve_options) # @option strict [Boolean] # @option via [Symbol] # @option env [Symbol, String] + # @option **resolve_options [Hash] # @return [void] # # @see Qonfig::Commands::Definition::ExposeVault # # @api public # @since 0.25.0 - def expose_vault(path, strict: true, via:, env:) + def expose_vault(path, strict: true, via:, env:, **file_resolve_options) definition_commands << Qonfig::Commands::Definition::ExposeVault.new( - path, strict: strict, via: via, env: env + path, strict: strict, via: via, env: env, file_resolve_options: file_resolve_options ) end end diff --git a/lib/qonfig/plugins/vault/loaders/vault.rb b/lib/qonfig/plugins/vault/loaders/vault.rb index 2af7a70c..f31c468f 100644 --- a/lib/qonfig/plugins/vault/loaders/vault.rb +++ b/lib/qonfig/plugins/vault/loaders/vault.rb @@ -20,8 +20,8 @@ class << self # # @api private # @since 0.25.0 - def load_file(path, fail_on_unexist: true, transform_values: true, version: nil) - data = load_data(path, version) + def load_file(path, fail_on_unexist: true, transform_values: true, version: nil, use_kv: true) + data = load_data(path, version, use_kv) raise Qonfig::FileNotFoundError, "Path #{path} not exist" if data == nil && fail_on_unexist result = data || empty_data return result unless transform_values @@ -49,17 +49,17 @@ def empty_data # # @api private # @since 0.25.1 - def load_data(file_path, version) + def load_data(file_path, version, use_kv) response = ::Vault.with_retries(::Vault::HTTPError) do - if version == nil - ::Vault.logical.read(file_path.to_s) - else + if use_kv mount_path, secret_path = file_path.to_s.split(::File::Separator, 2) ::Vault.kv(mount_path).read(secret_path, version) + else + ::Vault.logical.read(file_path.to_s) end end - response&.data&.dig(:data) + response&.data end # @param vault_data [Hash] diff --git a/spec/features/plugins/vault/expose_vault_spec.rb b/spec/features/plugins/vault/expose_vault_spec.rb index 6d995013..a580a825 100644 --- a/spec/features/plugins/vault/expose_vault_spec.rb +++ b/spec/features/plugins/vault/expose_vault_spec.rb @@ -11,17 +11,17 @@ end end let(:secret_data) do - { data: { production: { kek: 'pek', cheburek: true }, other_key: '<%= 1 + 1 %>' } } + { production: { kek: 'pek', cheburek: true }, other_key: '<%= 1 + 1 %>' } end let(:vault_class) do Class.new(Qonfig::DataSet) do setting :based_on_path do - expose_vault 'kv/data/path_based', via: :path, env: :production + expose_vault 'kv/data/path_based', via: :path, env: :production, use_kv: false end setting :based_on_env_key do - expose_vault 'kv/data/env_key', via: :env_key, env: 'production' + expose_vault 'kv/data/env_key', via: :env_key, env: 'production', use_kv: false end end end @@ -43,7 +43,7 @@ specify 'raises an error' do expect do Class.new(Qonfig::DataSet) do - expose_vault 'kv/data/path_based', via: Object.new, env: :production + expose_vault 'kv/data/path_based', via: Object.new, env: :production, use_kv: false end end.to raise_error(Qonfig::ArgumentError) end @@ -53,7 +53,7 @@ specify 'raises an error' do expect do Class.new(Qonfig::DataSet) do - expose_vault 'kv/data/path_based', via: :path, env: Object.new + expose_vault 'kv/data/path_based', via: :path, env: Object.new, use_kv: false end end.to raise_error(Qonfig::ArgumentError) end @@ -63,7 +63,7 @@ specify 'raises an error' do expect do Class.new(Qonfig::DataSet) do - expose_vault 'kv/data/path_based', via: :kek, env: :production + expose_vault 'kv/data/path_based', via: :kek, env: :production, use_kv: false end end.to raise_error(Qonfig::ArgumentError) end @@ -73,7 +73,7 @@ specify 'raises an error' do expect do Class.new(Qonfig::DataSet) do - expose_vault 'kv/data/path_based', via: :path, env: '' + expose_vault 'kv/data/path_based', via: :path, env: '', use_kv: false end end.to raise_error(Qonfig::ArgumentError) end @@ -82,7 +82,7 @@ context "when provided key doesn't exist" do let(:vault_class) do Class.new(Qonfig::DataSet) do - expose_vault 'kv/data/env_key', via: :env_key, env: 'kekduction' + expose_vault 'kv/data/env_key', via: :env_key, env: 'kekduction', use_kv: false end end @@ -97,7 +97,8 @@ let(:vault_class) do Class.new(Qonfig::DataSet) do setting :based_on_env_key do - expose_vault 'kv/data/env_key', via: :env_key, env: 'production', strict: false + expose_vault 'kv/data/env_key', + via: :env_key, env: 'production', strict: false, use_kv: false end end end diff --git a/spec/features/plugins/vault/load_from_vault_spec.rb b/spec/features/plugins/vault/load_from_vault_spec.rb index 510774ea..491dc16d 100644 --- a/spec/features/plugins/vault/load_from_vault_spec.rb +++ b/spec/features/plugins/vault/load_from_vault_spec.rb @@ -12,11 +12,11 @@ allow(instance).to receive(:data).and_return(secret_data) end end - let(:secret_data) { Hash[data: { kek: true, pek: 'cheburek', nested: Hash[key: 123] }] } + let(:secret_data) { Hash[kek: true, pek: 'cheburek', nested: { key: 123 }] } let(:vault_class) do Class.new(Qonfig::DataSet) do - load_from_vault 'kv/data/development' + load_from_vault 'kv/data/development', use_kv: false end end @@ -43,7 +43,7 @@ context 'with Pathname at path argument' do let(:vault_class) do Class.new(Qonfig::DataSet) do - load_from_vault Pathname('kv/data/development') + load_from_vault Pathname('kv/data/development'), use_kv: false end end @@ -57,7 +57,7 @@ context 'when strict set to false' do let(:vault_class) do Class.new(Qonfig::DataSet) do - load_from_vault 'kv/data/development', strict: false + load_from_vault 'kv/data/development', strict: false, use_kv: false end end @@ -82,7 +82,7 @@ context 'when version specified' do let(:vault_class) do Class.new(Qonfig::DataSet) do - load_from_vault 'kv/data/development', version: 2 + load_from_vault 'kv/data/development', version: 2, use_kv: true end end diff --git a/spec/features/plugins/vault/load_from_yaml_spec.rb b/spec/features/plugins/vault/load_from_yaml_spec.rb index 3d1b72d8..f8608646 100644 --- a/spec/features/plugins/vault/load_from_yaml_spec.rb +++ b/spec/features/plugins/vault/load_from_yaml_spec.rb @@ -13,24 +13,24 @@ end end - let(:secret_data) { Hash[data: { "file.yml": yaml_content }] } - let(:yaml_content) { YAML.dump(kek: "pek") } + let(:secret_data) { Hash["file.yml": yaml_content] } + let(:yaml_content) { YAML.dump(kek: 'pek') } let(:vault_class) do Class.new(Qonfig::DataSet) do - load_from_yaml 'vault://kv/data/development/file.yml' + load_from_yaml 'vault://kv/data/development/file.yml', use_kv: false end end specify 'defines config object by yaml instructions' do expect(Vault.logical).to receive(:read).with('kv/data/development').and_return(returned_data) VaultConfig.new.settings.tap do |conf| - expect(conf).to have_attributes(kek: "pek") + expect(conf).to have_attributes(kek: 'pek') end end context "when key doesn't exist" do - let(:secret_data) { Hash[data: { "other_file.yml": yaml_content }] } + let(:secret_data) { Hash[data: { 'other_file.yml': yaml_content }] } specify 'raises error' do expect(Vault.logical).to receive(:read).with('kv/data/development').and_return(returned_data) From f43964e21e6b8c14a6d48004510becfd29cc299f Mon Sep 17 00:00:00 2001 From: JustAnotherDude Date: Tue, 22 Dec 2020 02:58:47 +0300 Subject: [PATCH 9/9] Change version and fix doc --- CHANGELOG.md | 2 +- lib/qonfig.rb | 6 +++--- lib/qonfig/commands/definition/load_from_json.rb | 2 +- lib/qonfig/commands/definition/load_from_yaml.rb | 2 +- lib/qonfig/commands/instantiation/values_file.rb | 4 ++-- lib/qonfig/file_data_resolving.rb | 2 +- lib/qonfig/file_data_resolving/mixin.rb | 6 +++--- lib/qonfig/file_data_resolving/resolver.rb | 12 ++++++------ .../toml/commands/definition/load_from_toml.rb | 2 +- lib/qonfig/plugins/vault.rb | 2 +- .../vault/commands/definition/expose_vault.rb | 2 +- .../vault/commands/definition/load_from_vault.rb | 2 +- lib/qonfig/plugins/vault/loaders/vault.rb | 3 ++- lib/qonfig/version.rb | 2 +- 14 files changed, 25 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 075981db..eb7114b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog All notable changes to this project will be documented in this file. -## [0.25.0] - 2020-12-22 +## [0.26.0] - 2020-12-22 ### Added - Added file data resolver functionality: - Added resolver for `file` diff --git a/lib/qonfig.rb b/lib/qonfig.rb index 5de950de..83714792 100644 --- a/lib/qonfig.rb +++ b/lib/qonfig.rb @@ -33,7 +33,7 @@ module Qonfig extend Validation::PredefinitionMixin # @api public - # @since 0.25.1 + # @since 0.26.0 extend FileDataResolving::Mixin # @since 0.20.0 @@ -65,13 +65,13 @@ module Qonfig # @since 0.20.0 define_validator(:not_nil) { |value| !value.nil? } - # @since 0.25.1 + # @since 0.26.0 define_resolver(:file) do |file_path| ::File.read(file_path) rescue Errno::ENOENT => error raise Qonfig::FileNotFoundError, error.message end - # @since 0.25.1 + # @since 0.26.0 set_default_resolver :file # @since 0.12.0 diff --git a/lib/qonfig/commands/definition/load_from_json.rb b/lib/qonfig/commands/definition/load_from_json.rb index 93f859fa..3230837f 100644 --- a/lib/qonfig/commands/definition/load_from_json.rb +++ b/lib/qonfig/commands/definition/load_from_json.rb @@ -21,7 +21,7 @@ class Qonfig::Commands::Definition::LoadFromJSON < Qonfig::Commands::Base # @return [Hash] # # @api private - # @since 0.25.1 + # @since 0.26.0 attr_reader :file_resolve_options # @param file_path [String, Pathname] diff --git a/lib/qonfig/commands/definition/load_from_yaml.rb b/lib/qonfig/commands/definition/load_from_yaml.rb index 4b4c64cc..a1ec3833 100644 --- a/lib/qonfig/commands/definition/load_from_yaml.rb +++ b/lib/qonfig/commands/definition/load_from_yaml.rb @@ -21,7 +21,7 @@ class Qonfig::Commands::Definition::LoadFromYAML < Qonfig::Commands::Base # @return [Hash] # # @api private - # @since 0.25.1 + # @since 0.26.0 attr_reader :file_resolve_options # @param file_path [String, Pathname] diff --git a/lib/qonfig/commands/instantiation/values_file.rb b/lib/qonfig/commands/instantiation/values_file.rb index 599215e2..c457619c 100644 --- a/lib/qonfig/commands/instantiation/values_file.rb +++ b/lib/qonfig/commands/instantiation/values_file.rb @@ -64,7 +64,7 @@ class Qonfig::Commands::Instantiation::ValuesFile < Qonfig::Commands::Base # @return [Hash] # # @api private - # @since 0.25.1 + # @since 0.26.0 attr_reader :file_resolve_options # @param file_path [String, Symbol, Pathname] @@ -77,7 +77,7 @@ class Qonfig::Commands::Instantiation::ValuesFile < Qonfig::Commands::Base # # @api private # @since 0.17.0 - # @version 0.25.1 + # @version 0.26.0 def initialize( file_path, caller_location, diff --git a/lib/qonfig/file_data_resolving.rb b/lib/qonfig/file_data_resolving.rb index a4729a59..8a5b171f 100644 --- a/lib/qonfig/file_data_resolving.rb +++ b/lib/qonfig/file_data_resolving.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # @api private -# @since 0.25.1 +# @since 0.26.0 module Qonfig::FileDataResolving require_relative 'file_data_resolving/resolver' require_relative 'file_data_resolving/mixin' diff --git a/lib/qonfig/file_data_resolving/mixin.rb b/lib/qonfig/file_data_resolving/mixin.rb index 7f58ba04..39b8c58a 100644 --- a/lib/qonfig/file_data_resolving/mixin.rb +++ b/lib/qonfig/file_data_resolving/mixin.rb @@ -1,14 +1,14 @@ # frozen_string_literal: true # @api public -# @since 0.25.1 +# @since 0.26.0 module Qonfig::FileDataResolving::Mixin # @param scheme_name [Symbol,String] # @param block [Block] # @return [void] # # @api public - # @since 0.25.1 + # @since 0.26.0 def define_resolver(scheme_name, &block) Qonfig::FileDataResolving::Resolver.add_resolver!(scheme_name, block) end @@ -17,7 +17,7 @@ def define_resolver(scheme_name, &block) # @return [void] # # @api public - # @since 0.25.1 + # @since 0.26.0 def set_default_resolver(scheme_name) Qonfig::FileDataResolving::Resolver.set_default_resolver!(scheme_name) end diff --git a/lib/qonfig/file_data_resolving/resolver.rb b/lib/qonfig/file_data_resolving/resolver.rb index a149d1c4..51694a15 100644 --- a/lib/qonfig/file_data_resolving/resolver.rb +++ b/lib/qonfig/file_data_resolving/resolver.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # @api private -# @since 0.25.1 +# @since 0.26.0 class Qonfig::FileDataResolving::Resolver class << self # @param scheme [Symbol,String] @@ -9,7 +9,7 @@ class << self # @return [void] # # @api private - # @since 0.25.1 + # @since 0.26.0 def add_resolver!(scheme, resolver_proc) @resolvers ||= {} @resolvers[scheme.to_sym] = resolver_proc @@ -19,7 +19,7 @@ def add_resolver!(scheme, resolver_proc) # @return [void] # # @api private - # @since 0.25.1 + # @since 0.26.0 def set_default_resolver!(scheme_name) @default_resolver = resolvers.fetch(scheme_name.to_sym) end @@ -29,7 +29,7 @@ def set_default_resolver!(scheme_name) # @raise [Qonfig::FileNotFoundError] # # @api private - # @since 0.25.1 + # @since 0.26.0 def resolve!(file_path, **options) scheme_name = URI(file_path.to_s).scheme scheme_name = scheme_name.to_sym unless scheme_name == nil @@ -42,13 +42,13 @@ def resolve!(file_path, **options) # @return [Array] # # @api private - # @since 0.25.1 + # @since 0.26.0 attr_reader :resolvers # @return [Proc] # # @api private - # @since 0.25.1 + # @since 0.26.0 attr_reader :default_resolver end end diff --git a/lib/qonfig/plugins/toml/commands/definition/load_from_toml.rb b/lib/qonfig/plugins/toml/commands/definition/load_from_toml.rb index 6c95c457..f95d3078 100644 --- a/lib/qonfig/plugins/toml/commands/definition/load_from_toml.rb +++ b/lib/qonfig/plugins/toml/commands/definition/load_from_toml.rb @@ -22,7 +22,7 @@ class Qonfig::Commands::Definition::LoadFromTOML < Qonfig::Commands::Base # @return [Hash] # # @api private - # @since 0.25.1 + # @since 0.26.0 attr_reader :file_resolve_options # @param file_path [String] diff --git a/lib/qonfig/plugins/vault.rb b/lib/qonfig/plugins/vault.rb index 98b94d01..6d932168 100644 --- a/lib/qonfig/plugins/vault.rb +++ b/lib/qonfig/plugins/vault.rb @@ -27,7 +27,7 @@ def install! # @return [void] # - # @since 0.25.1 + # @since 0.26.0 # @api private def define_resolvers! ::Qonfig.define_resolver(:vault) do |file_path, **options| diff --git a/lib/qonfig/plugins/vault/commands/definition/expose_vault.rb b/lib/qonfig/plugins/vault/commands/definition/expose_vault.rb index 1bc129bc..7452fd85 100644 --- a/lib/qonfig/plugins/vault/commands/definition/expose_vault.rb +++ b/lib/qonfig/plugins/vault/commands/definition/expose_vault.rb @@ -45,7 +45,7 @@ class Qonfig::Commands::Definition::ExposeVault < Qonfig::Commands::Base # @return [Hash] # # @api private - # @since 0.25.1 + # @since 0.26.0 attr_reader :file_resolve_options # @param path [String Pathname] diff --git a/lib/qonfig/plugins/vault/commands/definition/load_from_vault.rb b/lib/qonfig/plugins/vault/commands/definition/load_from_vault.rb index e154ae08..575e8ce7 100644 --- a/lib/qonfig/plugins/vault/commands/definition/load_from_vault.rb +++ b/lib/qonfig/plugins/vault/commands/definition/load_from_vault.rb @@ -21,7 +21,7 @@ class Qonfig::Commands::Definition::LoadFromVault < Qonfig::Commands::Base # @return [Hash] # # @api private - # @since 0.25.1 + # @since 0.26.0 attr_reader :file_resolve_options # @param path [String] diff --git a/lib/qonfig/plugins/vault/loaders/vault.rb b/lib/qonfig/plugins/vault/loaders/vault.rb index f31c468f..1d525d61 100644 --- a/lib/qonfig/plugins/vault/loaders/vault.rb +++ b/lib/qonfig/plugins/vault/loaders/vault.rb @@ -14,6 +14,7 @@ class Qonfig::Loaders::Vault < Qonfig::Loaders::Basic class << self # @param path [String, Pathname] # @option fail_on_unexist [Boolean] + # @option version [String, Integer] # @return [Object] # # @raise [Qonfig::FileNotFoundError] @@ -48,7 +49,7 @@ def empty_data # @return [Object] # # @api private - # @since 0.25.1 + # @since 0.26.0 def load_data(file_path, version, use_kv) response = ::Vault.with_retries(::Vault::HTTPError) do if use_kv diff --git a/lib/qonfig/version.rb b/lib/qonfig/version.rb index 1bd31573..7cb1b178 100644 --- a/lib/qonfig/version.rb +++ b/lib/qonfig/version.rb @@ -5,5 +5,5 @@ module Qonfig # # @api public # @since 0.1.0 - VERSION = '0.25.1' + VERSION = '0.26.0' end