From c3ec865bcb37bf29792d247163ccb683a0a560c9 Mon Sep 17 00:00:00 2001 From: Fahmina Ghauri Date: Thu, 1 Apr 2021 00:45:57 +0500 Subject: [PATCH] changes to fetch flags from S3 - UL-916 --- unlaunchio/client/factory.go | 62 +++++++++++++-- unlaunchio/service/httpfeaturestore.go | 60 ++++++++++----- unlaunchio/service/httpfeaturestore_test.go | 4 +- unlaunchio/util/httpclient.go | 84 ++++++++++++++++++--- 4 files changed, 173 insertions(+), 37 deletions(-) diff --git a/unlaunchio/client/factory.go b/unlaunchio/client/factory.go index efd6828..da75843 100644 --- a/unlaunchio/client/factory.go +++ b/unlaunchio/client/factory.go @@ -2,12 +2,15 @@ package client import ( "errors" + "fmt" + "strings" + "sync/atomic" + "github.com/unlaunch/go-sdk/unlaunchio/engine" "github.com/unlaunch/go-sdk/unlaunchio/service" "github.com/unlaunch/go-sdk/unlaunchio/service/api" "github.com/unlaunch/go-sdk/unlaunchio/util" "github.com/unlaunch/go-sdk/unlaunchio/util/logger" - "strings" ) // UnlaunchFactory ... @@ -17,6 +20,8 @@ type UnlaunchFactory struct { logger logger.Interface } +var sync0Complete atomic.Value + // NewUnlaunchClientFactory is a factory func NewUnlaunchClientFactory(SDKKey string, cfg *UnlaunchClientConfig) (*UnlaunchFactory, error) { @@ -49,9 +54,25 @@ func (f *UnlaunchFactory) Client() Client { } } + if sync0Complete.Load() == nil { + sync0Complete.Store(true) // we preemptively marked it as done + client := f.sync0() + if client == nil { // regular server sync + return f.regularServerSync() + } else { + return client + } + + } else { + return f.regularServerSync() + } +} + +func (f *UnlaunchFactory) regularServerSync() Client { + eventsRecorder := api.NewHTTPEventsRecorder( false, - util.NewHTTPClient(f.sdkKey, f.cfg.Host, f.cfg.HTTPTimeout, f.logger), + util.NewHTTPClient(f.sdkKey, f.cfg.Host, f.cfg.HTTPTimeout, f.logger, false), "/api/v1/impressions", f.cfg.MetricsFlushInterval, f.cfg.MetricsQueueSize, @@ -59,23 +80,54 @@ func (f *UnlaunchFactory) Client() Client { f.logger) eventsCounts := api.NewEventsCountAggregator( - util.NewHTTPClient(f.sdkKey, f.cfg.Host, f.cfg.HTTPTimeout, f.logger), + util.NewHTTPClient(f.sdkKey, f.cfg.Host, f.cfg.HTTPTimeout, f.logger, false), "/api/v1/events", f.cfg.MetricsFlushInterval, f.cfg.MetricsQueueSize, f.logger) - hc := util.NewHTTPClient(f.sdkKey, f.cfg.Host, f.cfg.HTTPTimeout, f.logger) + hc := util.NewHTTPClient(f.sdkKey, f.cfg.Host, f.cfg.HTTPTimeout, f.logger, false) return &SimpleClient{ FeatureStore: service.NewHTTPFeatureStore( hc, f.cfg.PollingInterval, - f.logger), + f.logger, + false, + nil), eventsRecorder: eventsRecorder, eventsCountAggregator: eventsCounts, logger: f.logger, evaluator: engine.NewEvaluator(f.logger), } +} + +func (f *UnlaunchFactory) sync0() Client { + hc := util.NewHTTPClient(f.sdkKey, f.cfg.Host, f.cfg.HTTPTimeout, f.logger, true) + + res, err := hc.Get("https://app-qa-unlaunch-io-master-flags.s3-us-west-1.amazonaws.com") + + if res == nil { + return nil + } + + if err != nil { + f.logger.Error("[HTTP GET] error reading body", err) + return nil + } + + f.logger.Debug(fmt.Sprintf("[HTTP GET] data: %d", res)) + + return &SimpleClient{ + FeatureStore: service.NewHTTPFeatureStore( + hc, + f.cfg.PollingInterval, + f.logger, + true, + res), + logger: f.logger, + evaluator: engine.NewEvaluator(f.logger), + } + // return res } diff --git a/unlaunchio/service/httpfeaturestore.go b/unlaunchio/service/httpfeaturestore.go index f9ecf3d..efefb6a 100644 --- a/unlaunchio/service/httpfeaturestore.go +++ b/unlaunchio/service/httpfeaturestore.go @@ -4,11 +4,12 @@ import ( "encoding/json" "errors" "fmt" + "sort" + "time" + "github.com/unlaunch/go-sdk/unlaunchio/dtos" "github.com/unlaunch/go-sdk/unlaunchio/util" "github.com/unlaunch/go-sdk/unlaunchio/util/logger" - "sort" - "time" ) type HTTPFeatureStore struct { @@ -17,6 +18,8 @@ type HTTPFeatureStore struct { features map[string]dtos.Feature initialSyncComplete bool shutdownCh chan bool + sync0Complete bool + res []byte } func (h *HTTPFeatureStore) Shutdown() { @@ -25,30 +28,42 @@ func (h *HTTPFeatureStore) Shutdown() { } func (h *HTTPFeatureStore) fetchFlags() error { - res, err := h.httpClient.Get("/api/v1/flags") - - if err != nil { - if httpError, ok := err.(*dtos.HTTPError); ok { - if httpError.Code == 403 { - h.logger.Error( - fmt.Sprintf("The API key you provided was rejected by the server. %s", util.SDKKeyHelpMessage)) + var res []byte + var err error + if !h.sync0Complete { + res, err = h.httpClient.Get("/api/v1/flags") + + if err != nil { + if httpError, ok := err.(*dtos.HTTPError); ok { + if httpError.Code == 403 { + h.logger.Error( + fmt.Sprintf("The API key you provided was rejected by the server. %s", util.SDKKeyHelpMessage)) + } + } else { + h.logger.Error("error fetching flags ", err) + return err } - } else { - h.logger.Error("error fetching flags ", err) - return err } - } - if res == nil { - // No error and empty response means nothing changed - // most like due to 304; not modified - return nil + if res == nil { + // No error and empty response means nothing changed + // most like due to 304; not modified + return nil + } + + h.logger.Trace("responseDto ", string(res)) + + } else { + res = h.res } - h.logger.Trace("responseDto ", string(res)) + return h.initFeatureMap(res) +} + +func (h *HTTPFeatureStore) initFeatureMap(res []byte) error { var responseDto dtos.TopLevelEnvelope - err = json.Unmarshal(res, &responseDto) + err := json.Unmarshal(res, &responseDto) if err != nil { h.logger.Error("error parsing feature flag JSON response ", err) @@ -77,6 +92,7 @@ func (h *HTTPFeatureStore) fetchFlags() error { h.features = temp h.logger.Debug("Downloaded: ", len(h.features)) + return nil } @@ -115,12 +131,16 @@ func (h *HTTPFeatureStore) Ready(timeout time.Duration) { func NewHTTPFeatureStore( httpClient util.HTTPClient, pollingInterval time.Duration, - logger logger.Interface) FeatureStore { + logger logger.Interface, + sync0Complete bool, + res []byte) FeatureStore { httpStore := &HTTPFeatureStore{ httpClient: httpClient, logger: logger, initialSyncComplete: false, features: nil, + sync0Complete: sync0Complete, + res: res, } httpStore.shutdownCh = util.RunImmediatelyAndSchedule(httpStore.fetchFlags, pollingInterval) diff --git a/unlaunchio/service/httpfeaturestore_test.go b/unlaunchio/service/httpfeaturestore_test.go index a376e7b..be1c3fb 100644 --- a/unlaunchio/service/httpfeaturestore_test.go +++ b/unlaunchio/service/httpfeaturestore_test.go @@ -75,7 +75,7 @@ func TestWhen_DataIsReturned_Then_FeatureStoreIsReady(t *testing.T) { h := &mockHTTPClient{} h.returnValidJSON = true - fs := NewHTTPFeatureStore(h, 100000000, logger.NewLogger(nil)) + fs := NewHTTPFeatureStore(h, 100000000, logger.NewLogger(nil), false, nil) time.Sleep(100 * time.Millisecond) @@ -89,7 +89,7 @@ func getHTTPFeatureStore() FeatureStore { h := NewHTTPFeatureStore( &mockHTTPClient{}, 900, - logger.NewLogger(nil)) + logger.NewLogger(nil), false, nil) return h } diff --git a/unlaunchio/util/httpclient.go b/unlaunchio/util/httpclient.go index 9944f6a..04a4143 100644 --- a/unlaunchio/util/httpclient.go +++ b/unlaunchio/util/httpclient.go @@ -3,11 +3,12 @@ package util import ( "bytes" "fmt" - "github.com/unlaunch/go-sdk/unlaunchio/dtos" - "github.com/unlaunch/go-sdk/unlaunchio/util/logger" "io/ioutil" "net/http" "time" + + "github.com/unlaunch/go-sdk/unlaunchio/dtos" + "github.com/unlaunch/go-sdk/unlaunchio/util/logger" ) type HTTPClient interface { @@ -24,23 +25,45 @@ type simpleHTTPClient struct { lastModifiedAt string } +type GenericHTTPClient struct { + host string + httpClient *http.Client + logger logger.Interface + sdkKey string +} + // NewHTTPClient returns a new http client func NewHTTPClient( sdkKey string, host string, timeout time.Duration, logger logger.Interface, + sync0 bool, ) HTTPClient { - client := &http.Client{ - Timeout: timeout, - } + if sync0 { + client := &http.Client{ + Timeout: timeout, + } + + return &GenericHTTPClient{ + host: host, + httpClient: client, + logger: logger, + sdkKey: sdkKey, + } + + } else { + client := &http.Client{ + Timeout: timeout, + } - return &simpleHTTPClient{ - host: host, - httpClient: client, - logger: logger, - sdkKey: sdkKey, + return &simpleHTTPClient{ + host: host, + httpClient: client, + logger: logger, + sdkKey: sdkKey, + } } } @@ -126,3 +149,44 @@ func (c *simpleHTTPClient) Post(path string, body []byte) error { Msg: resp.Status, } } + +func (c *GenericHTTPClient) Get(path string) ([]byte, error) { + apiEndpoint := path + c.logger.Debug("[HTTP GET] ", apiEndpoint) + + req, _ := http.NewRequest("GET", apiEndpoint, nil) + + resp, err := c.httpClient.Do(req) + + if err != nil { + c.logger.Error("[HTTP GET] HTTP error ", err) + return nil, err + } + + defer resp.Body.Close() + + reader := resp.Body + defer reader.Close() + + body, err := ioutil.ReadAll(reader) + + if err != nil { + c.logger.Error("[HTTP GET] error reading body", err) + return nil, err + } + + c.logger.Debug(fmt.Sprintf("[HTTP GET] status code: %d", resp.StatusCode)) + + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return body, nil + } else { + return nil, &dtos.HTTPError{ + Code: resp.StatusCode, + Msg: resp.Status, + } + } +} + +func (c *GenericHTTPClient) Post(path string, body []byte) error { + return nil +}