Skip to content
This repository was archived by the owner on Oct 18, 2022. It is now read-only.

Commit 6b574b4

Browse files
author
venkatvghub
committed
[config] - Provides ability to read configs and secrets from environment variables
1 parent 6cee427 commit 6b574b4

4 files changed

Lines changed: 65 additions & 30 deletions

File tree

‎thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/util/DaoProviderUtil.java‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import org.apache.pinot.thirdeye.datalayer.bao.jdbc.AbstractManagerImpl;
3434
import org.apache.pinot.thirdeye.datalayer.dto.AbstractDTO;
3535
import org.apache.pinot.thirdeye.datalayer.util.PersistenceConfig.DatabaseConfiguration;
36+
import org.apache.pinot.thirdeye.util.CustomConfigReader;
3637
import org.apache.tomcat.jdbc.pool.DataSource;
3738
import org.h2.store.fs.FileUtils;
3839
import org.slf4j.Logger;
@@ -84,11 +85,18 @@ private static DataSource createDataSource(final PersistenceConfig configuration
8485
dataSource.setInitialSize(10);
8586
dataSource.setDefaultAutoCommit(false);
8687
dataSource.setMaxActive(100);
87-
dataSource.setUsername(configuration.getDatabaseConfiguration().getUser());
88-
dataSource.setPassword(configuration.getDatabaseConfiguration().getPassword());
89-
dataSource.setUrl(configuration.getDatabaseConfiguration().getUrl());
88+
89+
/* Read the database credentials and details from environment variables */
90+
CustomConfigReader ccr = new CustomConfigReader();
91+
String userName = ccr.readEnv(configuration.getDatabaseConfiguration().getUser());
92+
dataSource.setUsername(userName);
93+
String password = ccr.readEnv(configuration.getDatabaseConfiguration().getPassword());
94+
dataSource.setPassword(password);
95+
String url = ccr.readEnv(configuration.getDatabaseConfiguration().getUrl());
96+
dataSource.setUrl(url);
9097
dataSource.setDriverClassName(configuration.getDatabaseConfiguration().getDriver());
9198

99+
// Additional DB healthchecks and pool configuration. Can be removed
92100
dataSource.setValidationQuery("select 1");
93101
dataSource.setTestWhileIdle(true);
94102
dataSource.setTestOnBorrow(true);
@@ -102,10 +110,7 @@ private static DataSource createDataSource(final PersistenceConfig configuration
102110
return dataSource;
103111
}
104112

105-
public static void init(DataSource dataSource) {
106-
injector = Guice.createInjector(new ThirdEyePersistenceModule(dataSource));
107-
}
108-
113+
109114
public static PersistenceConfig readPersistenceConfig(File configFile) {
110115
YamlConfigurationFactory<PersistenceConfig> factory = new YamlConfigurationFactory<>(
111116
PersistenceConfig.class,

‎thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSourceConfig.java‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.commons.lang3.StringUtils;
2828
import org.apache.pinot.thirdeye.auto.onboard.AutoOnboardPinotMetadataSource;
2929
import org.apache.pinot.thirdeye.datasource.MetadataSourceConfig;
30+
import org.apache.pinot.thirdeye.util.CustomConfigReader;
3031
import org.slf4j.Logger;
3132
import org.slf4j.LoggerFactory;
3233

@@ -212,14 +213,16 @@ public PinotThirdEyeDataSourceConfig build() {
212213
"{} accepts only 'http' or 'https' connection schemes", className);
213214

214215
PinotThirdEyeDataSourceConfig config = new PinotThirdEyeDataSourceConfig();
215-
config.setControllerHost(controllerHost);
216+
// Read from the custom config reader for reading from environment variables
217+
CustomConfigReader ccr = new CustomConfigReader();
218+
config.setControllerHost(ccr.readEnv(controllerHost));
216219
config.setControllerPort(controllerPort);
217-
config.setZookeeperUrl(zookeeperUrl);
218-
config.setClusterName(clusterName);
219-
config.setBrokerUrl(brokerUrl);
220-
config.setTag(tag);
220+
config.setZookeeperUrl(ccr.readEnv(zookeeperUrl));
221+
config.setClusterName(ccr.readEnv(clusterName));
222+
config.setBrokerUrl(ccr.readEnv(brokerUrl));
223+
config.setTag(ccr.readEnv(tag));
221224
config.setControllerConnectionScheme(controllerConnectionScheme);
222-
config.setName(name);
225+
config.setName(ccr.readEnv(name));
223226
return config;
224227
}
225228
}

‎thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java‎

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.apache.pinot.thirdeye.datasource.pinot.resultset.ThirdEyeResultSetGroup;
4040
import org.apache.pinot.thirdeye.detection.ConfigUtils;
4141
import org.apache.pinot.thirdeye.util.ThirdEyeUtils;
42+
import org.apache.pinot.thirdeye.util.CustomConfigReader;
4243
import org.apache.tomcat.jdbc.pool.DataSource;
4344
import org.joda.time.DateTime;
4445
import org.joda.time.DateTimeZone;
@@ -85,22 +86,22 @@ public class SqlResponseCacheLoader extends CacheLoader<SqlQuery, ThirdEyeResult
8586
DataSource h2DataSource;
8687

8788
public SqlResponseCacheLoader(Map<String, Object> properties) throws Exception {
88-
89+
CustomConfigReader ccr = new CustomConfigReader();
8990
// Init Presto datasources
9091
if (properties.containsKey(PRESTO)) {
9192
List<Map<String, Object>> prestoMapList = ConfigUtils.getList(properties.get(PRESTO));
9293
for (Map<String, Object> objMap: prestoMapList) {
9394
Map<String, String> dbNameToURLMap = (Map)objMap.get(DB);
94-
String prestoUser = (String)objMap.get(USER);
95-
String prestoPassword = getPassword(objMap);
95+
String prestoUser = ccr.readEnv((String)objMap.get(USER));
96+
String prestoPassword = ccr.readEnv(getPassword(objMap));
9697

9798
for (Map.Entry<String, String> entry: dbNameToURLMap.entrySet()) {
9899
DataSource dataSource = new DataSource();
99100
dataSource.setInitialSize(INIT_CONNECTIONS);
100101
dataSource.setMaxActive(MAX_CONNECTIONS);
101102
dataSource.setUsername(prestoUser);
102103
dataSource.setPassword(prestoPassword);
103-
dataSource.setUrl(entry.getValue());
104+
dataSource.setUrl(ccr.readEnv(entry.getValue()));
104105

105106
// Timeout before an abandoned(in use) connection can be removed.
106107
dataSource.setRemoveAbandonedTimeout(ABANDONED_TIMEOUT);
@@ -117,16 +118,16 @@ public SqlResponseCacheLoader(Map<String, Object> properties) throws Exception {
117118
List<Map<String, Object>> mysqlMapList = ConfigUtils.getList(properties.get(MYSQL));
118119
for (Map<String, Object> objMap: mysqlMapList) {
119120
Map<String, String> dbNameToURLMap = (Map)objMap.get(DB);
120-
String mysqlUser = (String)objMap.get(USER);
121-
String mysqlPassword = getPassword(objMap);
121+
String mysqlUser = ccr.readEnv((String)objMap.get(USER));
122+
String mysqlPassword = ccr.readEnv(getPassword(objMap));
122123

123124
for (Map.Entry<String, String> entry: dbNameToURLMap.entrySet()) {
124125
DataSource dataSource = new DataSource();
125126
dataSource.setInitialSize(INIT_CONNECTIONS);
126127
dataSource.setMaxActive(MAX_CONNECTIONS);
127128
dataSource.setUsername(mysqlUser);
128129
dataSource.setPassword(mysqlPassword);
129-
dataSource.setUrl(entry.getValue());
130+
dataSource.setUrl(ccr.readEnv(entry.getValue()));
130131

131132
// Timeout before an abandoned(in use) connection can be removed.
132133
dataSource.setRemoveAbandonedTimeout(ABANDONED_TIMEOUT);
@@ -143,9 +144,9 @@ public SqlResponseCacheLoader(Map<String, Object> properties) throws Exception {
143144
List<Map<String, Object>> verticaMapList = ConfigUtils.getList(properties.get(VERTICA));
144145
for (Map<String, Object> objMap: verticaMapList) {
145146
Map<String, String> dbNameToURLMap = (Map)objMap.get(DB);
146-
String verticaUser = (String)objMap.get(USER);
147-
String verticaPassword = getPassword(objMap);
148-
String verticaDriver = (String)objMap.get(DRIVER);
147+
String verticaUser = ccr.readEnv((String)objMap.get(USER));
148+
String verticaPassword = ccr.readEnv(getPassword(objMap));
149+
String verticaDriver = ccr.readEnv((String)objMap.get(DRIVER));
149150

150151
for (Map.Entry<String, String> entry: dbNameToURLMap.entrySet()) {
151152
DataSource dataSource = new DataSource();
@@ -154,7 +155,7 @@ public SqlResponseCacheLoader(Map<String, Object> properties) throws Exception {
154155
dataSource.setUsername(verticaUser);
155156
dataSource.setPassword(verticaPassword);
156157
dataSource.setDriverClassName(verticaDriver);
157-
dataSource.setUrl(entry.getValue());
158+
dataSource.setUrl(ccr.readEnv(entry.getValue()));
158159

159160
// Timeout before an abandoned(in use) connection can be removed.
160161
dataSource.setRemoveAbandonedTimeout(ABANDONED_TIMEOUT);
@@ -170,16 +171,16 @@ public SqlResponseCacheLoader(Map<String, Object> properties) throws Exception {
170171
if (properties.containsKey(BIGQUERY)) {
171172
List<Map<String, Object>> bigQueryMapList = ConfigUtils.getList(properties.get(BIGQUERY));
172173
for (Map<String, Object> objMap: bigQueryMapList) {
173-
System.out.println(bigQueryMapList.toString());
174+
// System.out.println(bigQueryMapList.toString()); Looks like this might leak credentials. Removing this
174175
Map<String, String> dbNameToURLMap = (Map)objMap.get(DB);
175-
String bigQueryDriver = (String)objMap.get(DRIVER);
176+
String bigQueryDriver = ccr.readEnv((String)objMap.get(DRIVER));
176177

177178
for (Map.Entry<String, String> entry: dbNameToURLMap.entrySet()) {
178179
DataSource dataSource = new DataSource();
179180
dataSource.setInitialSize(INIT_CONNECTIONS);
180181
dataSource.setMaxActive(MAX_CONNECTIONS);
181182
dataSource.setDriverClassName(bigQueryDriver);
182-
dataSource.setUrl(entry.getValue());
183+
dataSource.setUrl(ccr.readEnv(entry.getValue()));
183184

184185
// Timeout before an abandoned(in use) connection can be removed.
185186
dataSource.setRemoveAbandonedTimeout(ABANDONED_TIMEOUT);
@@ -198,9 +199,9 @@ public SqlResponseCacheLoader(Map<String, Object> properties) throws Exception {
198199

199200
h2DataSource.setInitialSize(INIT_CONNECTIONS);
200201
h2DataSource.setMaxActive(MAX_CONNECTIONS);
201-
String h2User = (String) objMap.get(USER);
202-
String h2Password = getPassword(objMap);
203-
h2Url = (String) objMap.get(DB);
202+
String h2User = ccr.readEnv((String) objMap.get(USER));
203+
String h2Password = ccr.readEnv(getPassword(objMap));
204+
h2Url = ccr.readEnv((String) objMap.get(DB));
204205
h2DataSource.setUsername(h2User);
205206
h2DataSource.setPassword(h2Password);
206207
h2DataSource.setUrl(h2Url);

‎thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/util/CustomConfigReader.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package org.apache.pinot.thirdeye.util;
2+
3+
import java.util.regex.Matcher;
4+
import java.util.regex.Pattern;
5+
6+
public class CustomConfigReader {
7+
public String readEnv(String inputStr) {
8+
final String regex = "\\?\\$[A-Z0-9_]*";
9+
final Pattern pattern = Pattern.compile(regex);
10+
final Matcher matcher = pattern.matcher(inputStr);
11+
12+
while (matcher.find()) {
13+
String matchStr = matcher.group(0);
14+
String env = matchStr.substring(2);
15+
String value = System.getenv(env);
16+
if (value != null) {
17+
// replace original string with configured env variables
18+
inputStr = inputStr.replace(matchStr, value);
19+
} else {
20+
System.out.format("%s is" + " not assigned.%n", env);
21+
}
22+
}
23+
24+
return inputStr;
25+
}
26+
}

0 commit comments

Comments
 (0)