python.snippets

##-------------------------------------
## PANDAS
##-------------------------------------

snippet pd chunk
	chunksize = 10 ** 6
	for chunk in pd.read_csv(filename, chunksize=chunksize):
	    process(chunk)

##-------------------------------------
## TIDYVERSE
##-------------------------------------

snippet summarise
	## one var, one summary func
	df.groupby("group_var")["outcome_var"].mean()
	## one var, many summary funcs
	df.groupby("group_var").agg(["sum","count"])
	## multiple vars, single summary func (over row as series)
	df.groupby("group_var").agg(lambda row: "|".join(row.tolist()))
	## multiple vars, multiple summary funcs
	df.groupby("group_var").agg({"A":"sum", "B": lambda x: len(x.unique().tolist())})

snippet gather
	pd.melt(df, id_vars="A", value_vars="B")

snippet as_factor
	pd.Categorical(df["col"], categories=["No", "Yes"], ordered=True)

snippet left_join
	df_c = df_a \
		.merge(df_b, how = "left", on = "x")

snippet right_join
	df_c = df_a \
		.merge(df_b, how = "right", on = "x")

snippet full_join
	df_c = df_a \
		.merge(df_b, how = "outer", on = "x")

snippet inner_join
	df_c = df_a \
		.merge(df_b, how = "inner", on = "x")

snippet mutate
	df["b"] = df["a"].apply(lambda x: x+1, 1)

snippet recode
	df["a"].replace({ ## original values on LHS
		1 : "No high school",
	    2 : "High school",
	    3 : "Some college",
	    4 : "2-yr degree",
	    5 : "4-yr degree",
	    6 : "Post-grad"
	})

snippet drop_na
	df = df.dropna(subset=["col1", "col2"])
	df = df.dropna(axis=1) ## drop columns with any missing values
	df = df.dropna(axis=1, how="all") ## drop columns with all missing values

snippet case_when
	df['new_column'].map({ ## must be exhaustive
		"old1" : "new1",
		"old2" : "new2"
	})

snippet case_when.2
	df['new_column'].update({ ## in-place
		"old1" : "new1",
		"old2" : "new2"
	})

snippet case_when.3
	df['new_column'] = 
		np.where(df['col']<9, 'value1',
	        np.where(df['col']<12, 'value2',
	        np.where(df['col']<15, 'value3', 'value4')))

##-------------------------------------
## VIZ
##-------------------------------------

snippet mpl figure
	## Init figure
	w, h = (8.5, 5)
	fig = plt.figure(figsize=(w,h))
	nrow, ncol = (1, 1)
	gs1 = gs.GridSpec(nrow, ncol)
	
	fig.subplots_adjust(top=0.9, bottom=0.1, left=0.125, right=0.9, wspace=0.2, hspace=0.2)
	ax0 = fig.add_subplot(gs1[0])

snippet mpl header
	import seaborn as sns
	sns.set_style("white")
	import matplotlib as mpl
	import matplotlib.gridspec as gs
	import matplotlib.patches as mpatches
	import matplotlib.pyplot as plt

##-------------------------------------
## GLM
##-------------------------------------

snippet statsmodels mixedlogit
	## https://www.statsmodels.org/stable/generated/statsmodels.genmod.bayes_mixed_glm.BinomialBayesMixedGLM.html#statsmodels.genmod.bayes_mixed_glm.BinomialBayesMixedGLM

	# A binomial (logistic) random effects model with random intercepts for villages and random slopes for each year within each village:
	random = {"a": '0 + C(Village)', "b": '0 + C(Village)*year_cen'}
	model = BinomialBayesMixedGLM.from_formula('y ~ year_cen', random, data)
	result = model.fit_vb()

snippet statsmodels glm
	## https://www.statsmodels.org/stable/glm.html
	import statsmodels.api as sm
	
	data = sm.datasets.scotland.load()
	data.exog = sm.add_constant(data.exog)

	# Instantiate a gamma family model with the default link function.
	gamma_model = sm.GLM(data.endog, data.exog, family=sm.families.Gamma())

	gamma_results = gamma_model.fit()
	print(gamma_results.summary())

snippet statsmodels linreg
	## https://www.statsmodels.org/stable/index.html
	import numpy as np
	import statsmodels.api as sm
	import statsmodels.formula.api as smf

	# Load data
	dat = sm.datasets.get_rdataset("Guerry", "HistData").data

	# Fit regression model (using the natural log of one of the regressors)
	results = smf.ols('Lottery ~ Literacy + np.log(Pop1831)', data=dat).fit()

	# Inspect the results
	print(results.summary())



##-------------------------------------
## ML
##-------------------------------------

def code_interaction_variables(X, interactions):
    """ Code specified interaction terms.

        X: dummied version of a dataframe
        interactions: either list of variable tuples or list of R-style interaction term strings
    """
    if type(interactions) == str:
        interactions = [interactions]
    elif type(interactions) == list or type(interactions) == tuple:
        if type(interactions[0]) != list and type(interactions[0]) != tuple:
            interactions = [interactions]

    for x_x in interactions:
        if type(x_x) == str:
            x_x = x_x.split(":")
            x_x = [x for x in x_x if x in X.columns]
        elif type(x_x) == tuple or type(x_x) == list:
            x_x = [x for x in x_x if x in X.columns]
        x_x_col = ":".join(x_x)
        X[x_x_col] = 1
        for x_col in x_x:
            X[x_x_col] *= X[x_col]
    return(X)

snippet onehot
	pd.get_dummies(df, drop_first = "True").replace({False: 0, True: 1})

snippet sklearn classif simple
	## template for a simple classification workflow
	import numpy as np
	from sklearn.model_selection import train_test_split
	from sklearn.model_selection import cross_val_score

	from sklearn import datasets

	from sklearn import svm
	from sklearn.tree import DecisionTreeClassifier
	from sklearn.ensemble import RandomForestClassifier
	from sklearn.linear_model import LogisticRegression

	X, y = make_classification(
		n_samples=1000, n_features=4,
        n_informative=2, n_redundant=0,
        random_state=0, shuffle=False
    )

	X_train, X_test, y_train, y_test = train_test_split(
	    X, y, test_size=0.4, random_state=0
	)

	X_train.shape, y_train.shape
	X_test.shape, y_test.shape

	clf = svm.SVC(kernel='linear', C=1).fit(X_train, y_train)
	clf.score(X_test, y_test)
	cross_val_score(clf, X, y, cv=5)

	clf = DecisionTreeClassifier().fit(X_train, y_train)
	clf.score(X_test, y_test)

	clf = RandomForestClassifier(max_depth=2, random_state=0).fit(X_train, y_train)
	clf.score(X_test, y_test)
	cross_val_score(clf, X, y, cv=5)

	clf = LogisticRegression(penalty="l1", C=1).fit(X, y)
	clf.score(X_test, y_test)
	cross_val_score(clf, X, y, cv=5)

snippet sklearn classif data
	from sklearn.datasets import make_classification
	
	X, y = make_classification(
	    n_samples=500,
	    n_features=15,
	    n_informative=3,
	    n_redundant=2,
	    n_repeated=0,
	    n_classes=8,
	    n_clusters_per_class=1,
	    class_sep=0.8,
	    random_state=0,
	)

snippet sklearn gmm
	import numpy as np
	import matplotlib.pyplot as plt
	from matplotlib.colors import LogNorm
	from sklearn import mixture
	
	n_samples = 300
	
	# generate random sample, two components
	np.random.seed(0)
	
	# generate spherical data centered on (20, 20)
	shifted_gaussian = np.random.randn(n_samples, 2) + np.array([20, 20])
	
	# generate zero centered stretched Gaussian data
	C = np.array([[0.0, -0.7], [3.5, 0.7]])
	stretched_gaussian = np.dot(np.random.randn(n_samples, 2), C)
	
	# concatenate the two datasets into the final training set
	X_train = np.vstack([shifted_gaussian, stretched_gaussian])
	
	# fit a Gaussian Mixture Model with two components
	clf = mixture.GaussianMixture(n_components=2, covariance_type="full")
	clf.fit(X_train)
	
	# display predicted scores by the model as a contour plot
	x = np.linspace(-20.0, 30.0)
	y = np.linspace(-20.0, 40.0)
	X, Y = np.meshgrid(x, y)
	XX = np.array([X.ravel(), Y.ravel()]).T
	Z = -clf.score_samples(XX)
	Z = Z.reshape(X.shape)
	
	CS = plt.contour(
	    X, Y, Z, norm=LogNorm(vmin=1.0, vmax=1000.0), levels=np.logspace(0, 3, 10)
	)
	CB = plt.colorbar(CS, shrink=0.8, extend="both")
	plt.scatter(X_train[:, 0], X_train[:, 1], 0.8)
	
	plt.title("Negative log-likelihood predicted by a GMM")
	plt.axis("tight")
	plt.show()

snippet sklearn stack
	from sklearn.ensemble import StackingRegressor
	from sklearn.linear_model import RidgeCV
	
	estimators = [
	    ("Random Forest", rf_pipeline),
	    ("Lasso", lasso_pipeline),
	    ("Gradient Boosting", gbdt_pipeline),
	]
	
	stacking_regressor = StackingRegressor(estimators=estimators, final_estimator=RidgeCV())
	stacking_regressor

snippet sklearn preproc
	from sklearn.preprocessing import OneHotEncoder
	from sklearn.preprocessing import StandardScaler
	
	cat_linear_processor = OneHotEncoder(handle_unknown="ignore")
	num_linear_processor = make_pipeline(
	    StandardScaler(), SimpleImputer(strategy="mean", add_indicator=True)
	)
	
	linear_preprocessor = make_column_transformer(
	    (num_linear_processor, num_selector), (cat_linear_processor, cat_selector)
	)
	linear_preprocessor

snippet sklearn logit
	from sklearn.datasets import load_iris
	from sklearn.linear_model import LogisticRegression
	
	X, y = load_iris(return_X_y=True)
	
	## basic
	clf = LogisticRegression(penalty = None).fit(X, y)
	## penalty ['l1', 'l2', 'elasticnet']
	## C (inverse of regularization strength)
	clf = LogisticRegression(penalty = "l1", C = 1).fit(X, y)
	
	clf.predict(X[:2, :])
	clf.predict_proba(X[:2, :])
	clf.score(X, y)

snippet sklearn linreg
	from sklearn import linear_model
	import numpy as np
	
	## linear OLS
	reg = linear_model.LinearRegression()
	reg.fit([[0, 0], [1, 1], [2, 2]], [0, 1, 2])
	reg.coef_
	reg.predict([[1, 1]])
	
	## ridge regression (L2 penalty tends towards zero-valued shrinkage)
	reg = linear_model.Ridge(alpha=.5)
	reg.fit([[0, 0], [0, 0], [1, 1]], [0, .1, 1])
	reg.coef_
	reg.intercept_
	reg.predict([[1, 1]])
	
	## ridge regression 10-fold cross-validation
	reg = linear_model.RidgeCV(alphas=np.logspace(-6, 6, 13), cv=10)
	reg.fit([[0, 0], [0, 0], [1, 1]], [0, .1, 1])
	reg.alpha_
	reg.predict([[1, 1]])
	
	## lasso regression (L1 penalty tends towards non-zero shrinkage)
	reg = linear_model.Lasso(alpha=0.1)
	reg.fit([[0, 0], [1, 1]], [0, 1])
	reg.predict([[1, 1]])

##-------------------------------------
## BAYESIAN
##-------------------------------------

snippet cmdstanpy mcmc
	## https://cmdstanpy.readthedocs.io/en/stable-0.9.65/sample.html
	import os
	from cmdstanpy import cmdstan_path, CmdStanModel
	bernoulli_stan = os.path.join(cmdstan_path(), 'examples', 'bernoulli', 'bernoulli.stan')
	bernoulli_data = os.path.join(cmdstan_path(), 'examples', 'bernoulli', 'bernoulli.data.json')

	# instantiate, compile bernoulli model
	bernoulli_model = CmdStanModel(stan_file=bernoulli_stan)

	# run the NUTS-HMC sampler
	bern_fit = bernoulli_model.sample(data=bernoulli_data)
	bern_fit.draws().shape
	bern_fit.summary()	

snippet cmdstanpy mle
	## https://cmdstanpy.readthedocs.io/en/stable-0.9.65/optimize.html
	import os
	from cmdstanpy.model import CmdStanModel
	from cmdstanpy.utils import cmdstan_path

	# instantiate, compile bernoulli model
	bernoulli_path = os.path.join(cmdstan_path(), 'examples', 'bernoulli', 'bernoulli.stan')
	bernoulli_model = CmdStanModel(stan_file=bernoulli_path)

	# run CmdStan's optimize method, returns object `CmdStanMLE`
	bern_data = os.path.join(cmdstan_path(), 'examples', 'bernoulli', 'bernoulli.data.json')
	bern_mle = bernoulli_model.optimize(data=bernoulli_data)
	print(bern_mle.column_names)
	print(bern_mle.optimized_params_dict)

snippet cmdstanpy vb
	## https://cmdstanpy.readthedocs.io/en/stable-0.9.65/variational_bayes.html
	import os
	from cmdstanpy.model import CmdStanModel
	from cmdstanpy.utils import cmdstan_path

	# instantiate, compile bernoulli model
	bernoulli_path = os.path.join(cmdstan_path(), 'examples', 'bernoulli', 'bernoulli.stan')
	bernoulli_model = CmdStanModel(stan_file=bernoulli_path)

	# run CmdStan's variational inference method, returns object `CmdStanVB`
	bern_data = os.path.join(cmdstan_path(), 'examples', 'bernoulli', 'bernoulli.data.json')
	bern_vb = bernoulli_model.variational(data=bern_data)
	print(bern_vb.column_names)
	print(bern_vb.variational_params_dict)
	bern_vb.variational_sample.shape


snippet pystan ex
	import stan
	
	schools_code = """
	data {
	  int<lower=0> J;         // number of schools
	  real y[J];              // estimated treatment effects
	  real<lower=0> sigma[J]; // standard error of effect estimates
	}
	parameters {
	  real mu;                // population treatment effect
	  real<lower=0> tau;      // standard deviation in treatment effects
	  vector[J] eta;          // unscaled deviation from mu by school
	}
	transformed parameters {
	  vector[J] theta = mu + tau * eta;        // school treatment effects
	}
	model {
	  target += normal_lpdf(eta | 0, 1);       // prior log-density
	  target += normal_lpdf(y | theta, sigma); // log-likelihood
	}
	"""
	
	schools_data = {"J": 8,
	                "y": [28,  8, -3,  7, -1,  1, 18, 12],
	                "sigma": [15, 10, 16, 11,  9, 11, 10, 18]}
	
	posterior = stan.build(schools_code, data=schools_data)
	fit = posterior.sample(num_chains=4, num_samples=1000)
	eta = fit["eta"]  # array with shape (8, 4000)
	df = fit.to_frame()

snippet pystan2 ex
	## DEPRECATED (only for reference)
	import pystan
	
	schools_code = """
	data {
	    int<lower=0> J; // number of schools
	    real y[J]; // estimated treatment effects
	    real<lower=0> sigma[J]; // s.e. of effect estimates
	}
	parameters {
	    real mu;
	    real<lower=0> tau;
	    real eta[J];
	}
	transformed parameters {
	    real theta[J];
	    for (j in 1:J)
	    theta[j] = mu + tau * eta[j];
	}
	model {
	    eta ~ normal(0, 1);
	    y ~ normal(theta, sigma);
	}
	"""
	
	schools_dat = {'J': 8,
	               'y': [28,  8, -3,  7, -1,  1, 18, 12],
	               'sigma': [15, 10, 16, 11,  9, 11, 10, 18]}
	
	sm = pystan.StanModel(model_code=schools_code)
	fit = sm.sampling(data=schools_dat, iter=1000, chains=4)
	
	a = fit.extract(permuted=False)
	
	print(fit)
	fit.plot()

##-------------------------------------
## SCRAPING
##-------------------------------------

snippet attempts
	import datetime as dt
	import sys
	from tqdm import tqdm
	
	sleep = True
	attempts = 0
	while True:
	    try:
	        attempts += 1
	        $1 ## scraping code goes here 
	        break
	    except Exception as e:
	    ## what to do when rate-limited
	        print(e)
	        print(str(dt.datetime.today()))
	        print("\nAttempt %i failed..." % attempts)
	        if sleep:
	            _ = [time.sleep(1) for i in tqdm(range(60*60*6), desc="sleeping")]
	        else:
	            print("Options:\n")
	            print("*type `continue` to try again")
	            print("*type `skip` to skip")
	            print("*try sleeping via `time.sleep()`")
	            print("*exit via `sys.exit()`")
	            print("*execute some other code")
	            _input = input("\n>")
	            if "continue" in _input or _input.strip() == None:
	                continue
	            if "skip" in _input:
	                break
	            else:
	                try:
	                    eval(_input+"\n")
	                except Exception as e:
	                    print(e)
	                    continue

snippet beautifulsoup
	import requests
	from bs4 import BeautifulSoup
	
	response = requests.get('http://paris.quel-institut-beaute.com')
	soup = BeautifulSoup(response.content, 'lxml')
	
	# Find all tags with class 'ic'
	stores = soup.select('.ic')
	
	# Find all tags with class 'ic' and print text
	for div in soup.find_all("div", class_="ic"):
	    print(div.get_text(strip=True))

snippet selenium
	from selenium import webdriver
	from selenium.webdriver.common.keys import Keys
	from selenium.webdriver.support import expected_conditions as EC
	from selenium.webdriver.chrome.options import Options
	from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
	from selenium.webdriver.common.action_chains import ActionChains
	
	from webdriver_manager.chrome import ChromeDriverManager
	
	## helpers 
	def scroll_down(driver, x = 5):
	    html = driver.find_element_by_tag_name('html')
	    for i in range(x):
	        html.send_keys(Keys.PAGE_DOWN) 
	
	def click_on_element(driver, element, use_javascript = True):
	    scroll_to_element(driver, element)
	    if use_javascript:
	        driver.execute_script("arguments[0].click();", element)
	    else:
	        ActionChains(driver).move_to_element(element).click().perform()
	
	def scroll_to_element(driver, element):
	    desired_y = (element.size['height'] / 2) + element.location['y']
	    window_h = driver.execute_script('return window.innerHeight')
	    window_y = driver.execute_script('return window.pageYOffset')
	    current_y = (window_h / 2) + window_y
	    scroll_y_by = desired_y - current_y
	    driver.execute_script("window.scrollBy(0, arguments[0]);", scroll_y_by)
	
	## objects
	def FirefoxDriver(headless=1):
	    gecko_path = "./geckodriver"
	    binary = FirefoxBinary()
	    if headless:
	        binary.add_command_line_options('--headless')
	        os.environ['MOZ_HEADLESS'] = '1'
	    binary.add_command_line_options('--mute-audio')
	    # _ = [time.sleep(t) for t in tqdm(list(range(1,5)))]
	    return(webdriver.Firefox(executable_path=gecko_path, firefox_binary=binary, log_path="./geckodriver.log"))
	
	def ChromeDriver(headless=0):
	    chrome_path = "./chromedriver"
	    print("Running on Chrome")
	    driver_options = Options()  
	    driver_options.add_argument("--mute-audio") 
	    driver_options.add_argument("--start-maximized")
	    if headless:
	        driver_options.add_argument('--headless')
	    # return(webdriver.Chrome(executable_path=chrome_path, options=driver_options))
	    return(webdriver.Chrome(ChromeDriverManager().install(), options=driver_options))
	
	## instantiate driver
	# driver = FirefoxDriver(headless=1)
	driver = ChromeDriver(headless=1)
	driver.get(url)
	
	## example navigations
	tabs = driver.find_elements_by_tag_name("tab")
	my_tab = [tab for tab in tabs if tab.text == "My Tab"][0]
	click_on_element(driver, my_tab)
	
	driver.find_element_by_id("my_id").text
	
	tabs = driver.find_elements_by_xpath('//div[@node-type="tab"]')
	
	while True:
	    scroll_down(driver, 100) 
	    time.sleep(3)
	    all_info = driver.find_elements_by_id("info")

##-------------------------------------
## DEBUG
##-------------------------------------

snippet exitpoint
	# EXITPOINT >>>>>>>>>>>
	import sys; sys.exit(1)
	# <<<<<<<<<<<<<<<<<<<<<

snippet breakpoint
	# BREAKPOINT >>>>>>>>>>
	import pdb; pdb.set_trace()
	# <<<<<<<<<<<<<<<<<<<<<
	$0

snippet pause
	$0# PAUSE >>>>>>>>>>>>>>>>>>>>
	import time; time.sleep(1)
	# <<<<<<<<<<<<<<<<<<<<<<<<<<

snippet globals
	# =================================================
	# ==================== GLOBALS ====================
	# =================================================

snippet todo
	# TODO >>>>>>>>>>>>>>>>>>>>>>>>>>>
	# $0
	# <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<

snippet section
	## --------------------------------------------------------
	## $1
	## --------------------------------------------------------

snippet comment
	## --------------------------------------------------------
	## $1 
	## --------------------------------------------------------

snippet title
	# ============== $0 ==============

snippet settings
	# =================================================
	# ==================== HELPERS ====================
	# =================================================

snippet main
	# ==============================================
	# ==================== MAIN ====================
	# ==============================================

snippet timeit
	import datetime as dt
	time_start = dt.datetime.now() 
	$1 # code here
	time_elapsed = datetime.now() - time_start
	print 'Time elpased (hh:mm:ss.ms) {}'.format(time_elapsed)

snippet header
	"""
	Description of script.$1
	
	Author: Soubhik Barari
	
	Environment:
	- must use Python 2.7
	- must run on RCE
	
	Runtime:
	- Xh for ~y lines of inputs on May 28, 2018
	
	Input:
	- input.csv
	
	Output:
	- output on xxx.harvard.edu
	"""

##-------------------------------------
## SQL
##-------------------------------------

snippet hive header
	SET hive.exec.compress.intermediate=true;
	SET hive.exec.compress.output=false;
	SET hive.execution.engine=mr;
	SET hive.mapred.mode=nonstrict;
	SET hive.cli.print.header=true;
	SET hive.exec.dynamic.partition=true;
	SET hive.exec.dynamic.partition.mode=nonstrict;
	SET hive.enforce.bucketing = true;

snippet pyspark jdbc
	df = sqlContext.read.format('jdbc').options(url='jdbc:postgresql:dbserver', dbtable='schema.tablename').load()

snippet pyspark hive
	# sc is an existing SparkContext.
	from pyspark.sql import HiveContext
	sqlContext = HiveContext(sc)
	
	sqlContext.sql("CREATE TABLE IF NOT EXISTS src (key INT, value STRING)")
	sqlContext.sql("LOAD DATA LOCAL INPATH 'examples/src/main/resources/kv1.txt' INTO TABLE src")
	
	# Queries can be expressed in HiveQL.
	results = sqlContext.sql("FROM src SELECT key, value").toPandas()

snippet mysql ingest
	import threading
	
	MYSQL_USER = ""
	MYSQL_PASS = ""
	MYSQL_HOST = ""
	MYSQL_NAME = ""
	MYSQL_TBL  = ""
	BASE_CMD   = "mysql %s -u%s -p%s -e " % (MYSQL_NAME, MYSQL_USER, MYSQL_PASS) + "'%s'"
	FILE_PATHS = []
	
	def load_file(path_name):
	    load_subcmd = 'LOAD DATA LOCAL INFILE "%s" INTO TABLE %s.%s FIELDS TERMINATED BY "\t" LINES TERMINATED BY "\n" IGNORE 1 LINES;' % (MYSQL_NAME, MYSQL_TBL, path_name)
	    os.system(BASE_CMD % load_subcmd)
	
	# Create threads
	thread_list = []
	for file_path in FILE_PATHS:
	    t = threading.Thread(target=load_file, args=[file_path])
	    thread_list.append(t)
	
	# Begin threads
	for thread in thread_list:
	   thread.start()
	 
	# Clean up threads
	for thread in thread_list:
	   thread.join() 
	
	print("DONE.")

##-------------------------------------
## UNIX
##-------------------------------------

snippet screen
	screen_cmd = "screen -dmS {1} bash -c '{2}'".format(name, cmd)
	os.system(screen_cmd)

##-------------------------------------
## OTHER
##-------------------------------------

snippet secrets
	## Import secrets
	SECRETS_PATH = os.path.join(WD_PATH, "secrets")
	sys.path.append(SECRETS_PATH)
	
	# from access import * ## access credentials

snippet dropbox
	import dropbox
	
	dbx = dropbox.Dropbox(DBX_ACCESS_TOKEN)
	
	## list files
	db_dir = dbx.files_list_folder(db_dirpath, limit=2000)
	db_dirfiles = [fn.name for fn in db_dir.entries]
	while db_dir.has_more:
		db_dir = dbx.files_list_folder_continue(list_result.cursor)
		db_dirfiles += [fn.name for fn in db_dir.entries]
	
	## download
	with open(local_fpath, "wb") as f:
		metadata, res = dbx.files_download(db_fpath)
		f.write(res.content)
	
	## upload
	with open(local_fpath, "rb") as f:
		dbx.files_upload(f.read(), db_fpath, mute=True)

snippet import standard
	import argparse
	import csv
	import datetime as dt
	import json
	import os
	import re
	import sys
	import time

snippet import ml
	import numpy as np
	import pandas as pd
	import sklearn as skl
	import statsmodels.api as sm
	import cmdstanpy

snippet cache
	import pickle
	try:
		with open("file_name.pkl", "r") as file:
			obj = pickle.load(file)
		print "cache hit"
	except Exception as e:
		print e
		print "cache miss"
		with open("file_name.pkl", "w+") as file:
			pickle.dump(obj, file)

snippet random id
	import uuid
	str(uuid.uuid1())[:${1}]

snippet chunk
	import numpy as np
	lst = range(1,1000)
	n_chunks = 10
	chunks = np.array_split(lst, n_chunks)

snippet time stamp
	TIMESTAMP_YMD = dt.datetime.today().strftime('%Y-%m-%d')
	TIMESTAMP_UTC = dt.datetime.utcnow()
	TIMESTAMP_ISO = dt.datetime.now().isoformat()
	TIMESTAMP_SEC = time.time()

snippet time parse
	import datetime as dt
	
	date_obj = dt.datetime.strptime("01/01/2001", "%m/%d/%Y")
	date_obj.strftime("%B %d, %Y")

snippet timedelta
	import datetime as dt
	
	dt1 = dt.datetime.strptime(tstamp1, "%H:%M:%S.%f")
	dt2 = dt.datetime.strptime(tstamp2, "%H:%M:%S.%f")
	gap = (dt1 - dt2).total_seconds()

snippet date parse
	import dateutil
	dateutil.parser.parse("Today is Jan 5 2019", fuzzy=True)

snippet args
	parser = argparse.ArgumentParser(description='$0')
	
	parser.add_argument('--var', dest='var', nargs='?', default="",
	                     help='description of var.')
	parser.add_argument('--varList', dest='varList', nargs='*', default=[1,2,3],
	                    help='description of varList.')
	args = parser.parse_args()

snippet parallel
	from joblib import Parallel, delayed
	import time
	
	start = time.time()
	# single parameter job
	# process = jobs split over CPUs, more overhead 
	# thread = jobs share CPUs and RAM, less overhead
	Parallel(n_jobs=2, prefer='processes')(delayed(my_fun)(i) for i in range(10))
	
	# multi-parameter job
	Parallel(n_jobs=2, prefer='processes')(delayed(my_fun_2p)(i, j) for i in range(i_num) for j in range(j_num))
	end = time.time()
	
	print('{:.4f} s'.format(end-start))

snippet regex
	import re
	
	# re.match() checks for a match only at the beginning of the string, 
	# while re.search() checks for a match anywhere in the string 
	# (this is what Perl does by default).
	
	s = re.search("c", "abc")
	m = re.match("c", "abc")

snippet temp
	# DELETE >>>>>>>>>>>>>>
	$0
	# <<<<<<<<<<<<<<<<<<<<<

snippet path join
	os.path.join($1)

snippet path curr
	CURR_PATH =  os.path.dirname(os.getcwd())

snippet path wd
	idx = os.getcwd().lower().split("/").index("Name_of_base_directory$1")+1
	WD_PATH = "/".join(os.getcwd().split("/")[:idx])