diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8376a1a..8e0a287 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: ['3.10'] steps: - uses: actions/checkout@v2 diff --git a/notebookUtils.py b/notebookUtils.py index 9c6f89a..8917ea8 100644 --- a/notebookUtils.py +++ b/notebookUtils.py @@ -37,14 +37,13 @@ def retrySession( return session -def saveFile(file_path, sandbox_id, token, log, tags=None): +def saveFile(file_path, sandbox_id, log, tags=None): """ Construct a requests POST call with args and kwargs and process the results. Args: file_path: The relative path to the file from the datadir, including filename and extension sandbox_id: Id of the sandbox - token: Keboola Storage token log: Logger instance tags: Additional tags for the file Returns: @@ -57,10 +56,10 @@ def saveFile(file_path, sandbox_id, token, log, tags=None): if tags is None: tags = [] if 'DATA_LOADER_API_URL' in os.environ and os.environ['DATA_LOADER_API_URL']: - url = 'http://' + os.environ['DATA_LOADER_API_URL'] + '/data-loader-api/save' + url = 'http://' + os.environ['DATA_LOADER_API_URL'] + '/data-loader-api/internal/save' else: - url = 'http://data-loader-api/data-loader-api/save' - headers = {'X-StorageApi-Token': token, 'User-Agent': 'Keboola Sandbox Autosave Request'} + url = 'http://data-loader-api/data-loader-api/internal/save' + headers = {'Content-Type': 'application/json', 'User-Agent': 'Keboola Sandbox Autosave Request'} payload = {'file': {'source': os.path.relpath(file_path), 'tags': ['autosave', 'sandbox-' + sandbox_id] + tags}} # the timeout is set to > 3min because of the delay on 400 level exception responses @@ -98,20 +97,6 @@ def updateApiTimestamp(sandbox_id, log): log.error('Saving autosave to Sandboxes API errored: ' + result.text) -def getStorageTokenFromEnv(log): - """ - Find Keboola token in env vars - Args: - log: Logger instance - """ - - if 'KBC_TOKEN' in os.environ: - return os.environ['KBC_TOKEN'] - else: - log.error('Could not find Keboola Storage API token.') - raise Exception('Could not find Keboola Storage API token.') - - def compressFolder(folder_path): """ Gzip folder @@ -126,19 +111,18 @@ def compressFolder(folder_path): return gz_path -def saveFolder(folder_path, sandbox_id, token, log): +def saveFolder(folder_path, sandbox_id, log): """ Gzip folder and save it to Keboola Storage Args: folder_path: Path to the folder sandbox_id: Id of the sandbox - token: Keboola Storage token log: Logger instance """ if os.path.exists(folder_path): gz_path = compressFolder(folder_path) try: - saveFile(gz_path, sandbox_id, token, log, ['git']) + saveFile(gz_path, sandbox_id, log, ['git']) finally: if os.path.exists(gz_path): os.remove(gz_path) @@ -156,13 +140,12 @@ def scriptPostSave(model, os_path, contents_manager, **kwargs): log = contents_manager.log sandbox_id = os.environ['SANDBOX_ID'] - token = getStorageTokenFromEnv(log) updateApiTimestamp(sandbox_id, log) has_persistent_storage = os.getenv('HAS_PERSISTENT_STORAGE', 'False').lower() in ('true', '1') if not has_persistent_storage: - saveFile(os_path, sandbox_id, token, log) - saveFolder('/data/.git', sandbox_id, token, log) + saveFile(os_path, sandbox_id, log) + saveFolder('/data/.git', sandbox_id, log) def notebookSetup(c): diff --git a/requirements.txt b/requirements.txt index d4b5763..03f0228 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ pycodestyle==2.6.0 pyflakes==2.2.0 Pygments==2.7.4 pyparsing==2.4.7 -pytest==5.4.3 +pytest==6.2.5 requests==2.25.1 requests-mock==1.8.0 six==1.15.0 diff --git a/setup.py b/setup.py index bcea3ee..66c6489 100644 --- a/setup.py +++ b/setup.py @@ -2,9 +2,10 @@ setup( name='keboola-sandboxes-notebook-utils', - version='2.2.0', + version='2.3.0', url='https://github.com/keboola/sandboxes-notebook-utils', packages=['keboola_notebook_utils'], package_dir={'keboola_notebook_utils': ''}, - requires=['pip'] + python_requires='>=3.10', + install_requires=['requests'], ) diff --git a/tests/notebookUtils.py b/tests/notebookUtils.py index 74174b8..66d5a58 100644 --- a/tests/notebookUtils.py +++ b/tests/notebookUtils.py @@ -7,7 +7,7 @@ import string import tempfile -from notebookUtils import compressFolder, getStorageTokenFromEnv, notebookSetup, saveFile, saveFolder, \ +from notebookUtils import compressFolder, notebookSetup, saveFile, saveFolder, \ scriptPostSave, updateApiTimestamp def generate_random_string(): @@ -41,11 +41,10 @@ def test_scriptPostSave(self): with requests_mock.Mocker() as m: os.environ['SANDBOX_ID'] = '123' os.environ['DATA_LOADER_API_URL'] = 'dataloader' - os.environ['KBC_TOKEN'] = 'token' if 'HAS_PERSISTENT_STORAGE' in os.environ: del os.environ['HAS_PERSISTENT_STORAGE'] - dataLoaderMock = m.post('http://dataloader/data-loader-api/save', json={'result': 'ok'}) + dataLoaderMock = m.post('http://dataloader/data-loader-api/internal/save', json={'result': 'ok'}) dataLoaderActivityMock = m.post('http://dataloader/data-loader-api/internal/activity', json={'result': 'ok'}) contentsManager = type('', (), {})() @@ -62,9 +61,8 @@ def test_scriptPostSave_disabledPersistentStorage(self): with requests_mock.Mocker() as m: os.environ['SANDBOX_ID'] = '123' os.environ['DATA_LOADER_API_URL'] = 'dataloader' - os.environ['KBC_TOKEN'] = 'token' os.environ['HAS_PERSISTENT_STORAGE'] = '0' - dataLoaderMock = m.post('http://dataloader/data-loader-api/save', json={'result': 'ok'}) + dataLoaderMock = m.post('http://dataloader/data-loader-api/internal/save', json={'result': 'ok'}) dataLoaderActivityMock = m.post('http://dataloader/data-loader-api/internal/activity', json={'result': 'ok'}) contentsManager = type('', (), {})() @@ -77,14 +75,12 @@ def test_scriptPostSave_disabledPersistentStorage(self): assert dataLoaderActivityMock.call_count == 1 - def test_scriptPostSave_enabledPersistentStorage(self): with requests_mock.Mocker() as m: os.environ['SANDBOX_ID'] = '123' os.environ['DATA_LOADER_API_URL'] = 'dataloader' - os.environ['KBC_TOKEN'] = 'token' os.environ['HAS_PERSISTENT_STORAGE'] = '1' - dataLoaderMock = m.post('http://dataloader/data-loader-api/save', json={'result': 'ok'}) + dataLoaderMock = m.post('http://dataloader/data-loader-api/internal/save', json={'result': 'ok'}) dataLoaderActivityMock = m.post('http://dataloader/data-loader-api/internal/activity', json={'result': 'ok'}) contentsManager = type('', (), {})() @@ -94,19 +90,11 @@ def test_scriptPostSave_enabledPersistentStorage(self): assert dataLoaderMock.call_count == 0 assert dataLoaderActivityMock.call_count == 1 - def test_getStorageTokenFromEnvMissing(self): - os.environ.pop('KBC_TOKEN') - with pytest.raises(Exception): - getStorageTokenFromEnv(logging) - - def test_getStorageTokenFromEnvOk(self): - token = generate_random_string() - os.environ['KBC_TOKEN'] = token - assert getStorageTokenFromEnv(logging) == token - def test_updateApiTimestamp(self): with requests_mock.Mocker() as m: - dataLoaderActivityMock = m.post('http://dataloader/data-loader-api/internal/activity', json={'result': 'ok'}) + os.environ['DATA_LOADER_API_URL'] = 'dataloader' + url = 'http://dataloader/data-loader-api/internal/activity' + dataLoaderActivityMock = m.post(url, json={'result': 'ok'}) updateApiTimestamp('123', logging) @@ -115,9 +103,9 @@ def test_updateApiTimestamp(self): def test_saveFile(self): with requests_mock.Mocker() as m: os.environ['DATA_LOADER_API_URL'] = 'dataloader' - dataLoaderMock = m.post('http://dataloader/data-loader-api/save', json={'result': 'ok'}) + dataLoaderMock = m.post('http://dataloader/data-loader-api/internal/save', json={'result': 'ok'}) - saveFile('/file/path', '123', 'token', logging) + saveFile('/file/path', '123', logging) assert dataLoaderMock.call_count == 1 response = json.loads(dataLoaderMock.last_request.text) @@ -144,14 +132,14 @@ def test_compressFolder(self): def test_saveFolder(self): with requests_mock.Mocker() as m: os.environ['DATA_LOADER_API_URL'] = 'dataloader' - dataLoaderMock = m.post('http://dataloader/data-loader-api/save', json={'result': 'ok'}) + dataLoaderMock = m.post('http://dataloader/data-loader-api/internal/save', json={'result': 'ok'}) folder_prepare = tempfile.mkdtemp() + '/.git' os.mkdir(folder_prepare) f = open(folder_prepare + '/file.txt', 'a') f.write('content') f.close() - saveFolder(folder_prepare, '123', 'token', logging) + saveFolder(folder_prepare, '123', logging) assert dataLoaderMock.call_count == 1 response = json.loads(dataLoaderMock.last_request.text)