From c26c97e08211a4b8bdb2980b1a64bf0b84006d86 Mon Sep 17 00:00:00 2001 From: Erik Zigo Date: Tue, 24 Feb 2026 21:31:55 +0100 Subject: [PATCH 1/4] refactor: use data-loader-api internal endpoints, remove KBC_TOKEN dependency Co-Authored-By: Claude Opus 4.6 --- notebookUtils.py | 52 ++++++++++++++---------------------------- setup.py | 2 +- tests/notebookUtils.py | 38 ++++++++++-------------------- 3 files changed, 30 insertions(+), 62 deletions(-) diff --git a/notebookUtils.py b/notebookUtils.py index 6c50943..3589951 100644 --- a/notebookUtils.py +++ b/notebookUtils.py @@ -1,5 +1,3 @@ -from datetime import datetime -import json import os import sys from IPython.lib import passwd @@ -38,14 +36,13 @@ def retrySession( return session -def saveFile(file_path, sandbox_id, token, log, tags=None): +def saveFile(file_path, sandbox_id, log, tags=None): """ Construct a requests POST call with args and kwargs and process the results. Args: file_path: The relative path to the file from the datadir, including filename and extension sandbox_id: Id of the sandbox - token: Keboola Storage token log: Logger instance tags: Additional tags for the file Returns: @@ -58,10 +55,10 @@ def saveFile(file_path, sandbox_id, token, log, tags=None): if tags is None: tags = [] if 'DATA_LOADER_API_URL' in os.environ and os.environ['DATA_LOADER_API_URL']: - url = 'http://' + os.environ['DATA_LOADER_API_URL'] + '/data-loader-api/save' + url = 'http://' + os.environ['DATA_LOADER_API_URL'] + '/data-loader-api/internal/save' else: - url = 'http://data-loader-api/data-loader-api/save' - headers = {'X-StorageApi-Token': token, 'User-Agent': 'Keboola Sandbox Autosave Request'} + url = 'http://data-loader-api/data-loader-api/internal/save' + headers = {'Content-Type': 'application/json', 'User-Agent': 'Keboola Sandbox Autosave Request'} payload = {'file': {'source': os.path.relpath(file_path), 'tags': ['autosave', 'sandbox-' + sandbox_id] + tags}} # the timeout is set to > 3min because of the delay on 400 level exception responses @@ -75,43 +72,30 @@ def saveFile(file_path, sandbox_id, token, log, tags=None): raise e -def updateApiTimestamp(sandbox_id, token, log): +def updateApiTimestamp(sandbox_id, log): """ - Update autosave timestamp in Sandboxes API + Update autosave timestamp via data-loader-api Args: sandbox_id: Id of the sandbox - token: Keboola Storage token log: Logger instance """ headers = { 'Content-Type': 'application/json', 'User-Agent': 'Keboola Sandbox Autosave Request', - 'X-StorageApi-Token': token, } - url = os.environ['SANDBOXES_API_URL'] + '/sandboxes/' + sandbox_id - body = json.dumps({'lastAutosaveTimestamp': datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%f')}) - result = retrySession().patch(url, data=body, headers=headers) + if 'DATA_LOADER_API_URL' in os.environ and os.environ['DATA_LOADER_API_URL']: + url = 'http://' + os.environ['DATA_LOADER_API_URL'] + '/data-loader-api/internal/activity' + else: + url = 'http://data-loader-api/data-loader-api/internal/activity' + + result = retrySession().post(url, headers=headers) if result.status_code == requests.codes.ok: log.info('Successfully saved autosave to Sandboxes API') else: log.error('Saving autosave to Sandboxes API errored: ' + result.text) -def getStorageTokenFromEnv(log): - """ - Find Keboola token in env vars - Args: - log: Logger instance - """ - - if 'KBC_TOKEN' in os.environ: - return os.environ['KBC_TOKEN'] - else: - log.error('Could not find Keboola Storage API token.') - raise Exception('Could not find Keboola Storage API token.') - - def compressFolder(folder_path): """ Gzip folder @@ -126,19 +110,18 @@ def compressFolder(folder_path): return gz_path -def saveFolder(folder_path, sandbox_id, token, log): +def saveFolder(folder_path, sandbox_id, log): """ Gzip folder and save it to Keboola Storage Args: folder_path: Path to the folder sandbox_id: Id of the sandbox - token: Keboola Storage token log: Logger instance """ if os.path.exists(folder_path): gz_path = compressFolder(folder_path) try: - saveFile(gz_path, sandbox_id, token, log, ['git']) + saveFile(gz_path, sandbox_id, log, ['git']) finally: if os.path.exists(gz_path): os.remove(gz_path) @@ -156,10 +139,9 @@ def scriptPostSave(model, os_path, contents_manager, **kwargs): log = contents_manager.log sandbox_id = os.environ['SANDBOX_ID'] - token = getStorageTokenFromEnv(log) - saveFile(os_path, sandbox_id, token, log) - updateApiTimestamp(sandbox_id, token, log) - saveFolder('/data/.git', sandbox_id, token, log) + updateApiTimestamp(sandbox_id, log) + saveFile(os_path, sandbox_id, log) + saveFolder('/data/.git', sandbox_id, log) def notebookSetup(c): diff --git a/setup.py b/setup.py index df4e6df..4658562 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='keboola-sandboxes-notebook-utils', - version='1.3.2', + version='1.4.0.dev1', url='https://github.com/keboola/sandboxes-notebook-utils', packages=['keboola_notebook_utils'], package_dir={'keboola_notebook_utils': ''}, diff --git a/tests/notebookUtils.py b/tests/notebookUtils.py index ba48b7d..f3e9eb1 100644 --- a/tests/notebookUtils.py +++ b/tests/notebookUtils.py @@ -7,7 +7,7 @@ import string import tempfile -from notebookUtils import compressFolder, getStorageTokenFromEnv, notebookSetup, saveFile, saveFolder, \ +from notebookUtils import compressFolder, notebookSetup, saveFile, saveFolder, \ scriptPostSave, updateApiTimestamp def generate_random_string(): @@ -36,12 +36,10 @@ def test_notebookSetup(self): def test_scriptPostSave(self): with requests_mock.Mocker() as m: - os.environ['SANDBOXES_API_URL'] = 'http://sandboxes-api' os.environ['SANDBOX_ID'] = '123' os.environ['DATA_LOADER_API_URL'] = 'dataloader' - os.environ['KBC_TOKEN'] = 'token' - dataLoaderMock = m.post('http://dataloader/data-loader-api/save', json={'result': 'ok'}) - apiMock = m.patch('http://sandboxes-api/sandboxes/123', json={'result': 'ok'}) + dataLoaderMock = m.post('http://dataloader/data-loader-api/internal/save', json={'result': 'ok'}) + activityMock = m.post('http://dataloader/data-loader-api/internal/activity', json={'result': 'ok'}) contentsManager = type('', (), {})() contentsManager.log = logging @@ -51,35 +49,23 @@ def test_scriptPostSave(self): assert 'file' in dataLoaderMock.last_request.text assert 'tags' in dataLoaderMock.last_request.text - assert apiMock.call_count == 1 - assert 'lastAutosaveTimestamp' in apiMock.last_request.text - - def test_getStorageTokenFromEnvMissing(self): - os.environ.pop('KBC_TOKEN') - with pytest.raises(Exception): - getStorageTokenFromEnv(logging) - - def test_getStorageTokenFromEnvOk(self): - token = generate_random_string() - os.environ['KBC_TOKEN'] = token - assert getStorageTokenFromEnv(logging) == token + assert activityMock.call_count == 1 def test_updateApiTimestamp(self): with requests_mock.Mocker() as m: - os.environ['SANDBOXES_API_URL'] = 'http://sandboxes-api' - apiMock = m.patch('http://sandboxes-api/sandboxes/123', json={'result': 'ok'}) + os.environ['DATA_LOADER_API_URL'] = 'dataloader' + activityMock = m.post('http://dataloader/data-loader-api/internal/activity', json={'result': 'ok'}) - updateApiTimestamp('123', 'token', logging) + updateApiTimestamp('123', logging) - assert apiMock.call_count == 1 - assert 'lastAutosaveTimestamp' in apiMock.last_request.text + assert activityMock.call_count == 1 def test_saveFile(self): with requests_mock.Mocker() as m: os.environ['DATA_LOADER_API_URL'] = 'dataloader' - dataLoaderMock = m.post('http://dataloader/data-loader-api/save', json={'result': 'ok'}) + dataLoaderMock = m.post('http://dataloader/data-loader-api/internal/save', json={'result': 'ok'}) - saveFile('/file/path', '123', 'token', logging) + saveFile('/file/path', '123', logging) assert dataLoaderMock.call_count == 1 response = json.loads(dataLoaderMock.last_request.text) @@ -106,14 +92,14 @@ def test_compressFolder(self): def test_saveFolder(self): with requests_mock.Mocker() as m: os.environ['DATA_LOADER_API_URL'] = 'dataloader' - dataLoaderMock = m.post('http://dataloader/data-loader-api/save', json={'result': 'ok'}) + dataLoaderMock = m.post('http://dataloader/data-loader-api/internal/save', json={'result': 'ok'}) folder_prepare = tempfile.mkdtemp() + '/.git' os.mkdir(folder_prepare) f = open(folder_prepare + '/file.txt', 'a') f.write('content') f.close() - saveFolder(folder_prepare, '123', 'token', logging) + saveFolder(folder_prepare, '123', logging) assert dataLoaderMock.call_count == 1 response = json.loads(dataLoaderMock.last_request.text) From 1eba799414e8e96b8466c4e15aedb05c7497182b Mon Sep 17 00:00:00 2001 From: Erik Zigo Date: Tue, 24 Feb 2026 22:05:12 +0100 Subject: [PATCH 2/4] chore: restore CI python version to 3.8 Co-Authored-By: Claude Opus 4.6 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8376a1a..46c59c3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: ['3.8'] steps: - uses: actions/checkout@v2 From e092524b1f3a142d6fa4fe4013ac858fe0408ea4 Mon Sep 17 00:00:00 2001 From: Erik Zigo Date: Wed, 25 Feb 2026 17:32:57 +0100 Subject: [PATCH 3/4] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4658562..1f7ad26 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='keboola-sandboxes-notebook-utils', - version='1.4.0.dev1', + version='1.4.0, url='https://github.com/keboola/sandboxes-notebook-utils', packages=['keboola_notebook_utils'], package_dir={'keboola_notebook_utils': ''}, From 437748c3d0f0e7d0074b67f7ac790c4a8bd7891a Mon Sep 17 00:00:00 2001 From: Erik Zigo Date: Wed, 25 Feb 2026 19:11:28 +0100 Subject: [PATCH 4/4] Apply suggestion from @ErikZigo --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1f7ad26..9396cf7 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='keboola-sandboxes-notebook-utils', - version='1.4.0, + version='1.4.0', url='https://github.com/keboola/sandboxes-notebook-utils', packages=['keboola_notebook_utils'], package_dir={'keboola_notebook_utils': ''},