refactor: use data-loader-api internal endpoints, remove KBC_TOKEN

.github/workflows/test.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ['3.8']
 
     steps:
     - uses: actions/checkout@v2

notebookUtils.py

@@ -1,5 +1,3 @@
-from datetime import datetime
-import json
 import os
 import sys
 from IPython.lib import passwd
@@ -38,14 +36,13 @@ def retrySession(
     return session
 
 
-def saveFile(file_path, sandbox_id, token, log, tags=None):
+def saveFile(file_path, sandbox_id, log, tags=None):
     """
     Construct a requests POST call with args and kwargs and process the
     results.
     Args:
         file_path: The relative path to the file from the datadir, including filename and extension
         sandbox_id: Id of the sandbox
-        token: Keboola Storage token
         log: Logger instance
         tags: Additional tags for the file
     Returns:
@@ -58,10 +55,10 @@ def saveFile(file_path, sandbox_id, token, log, tags=None):
     if tags is None:
         tags = []
     if 'DATA_LOADER_API_URL' in os.environ and os.environ['DATA_LOADER_API_URL']:
-        url = 'http://' + os.environ['DATA_LOADER_API_URL'] + '/data-loader-api/save'
+        url = 'http://' + os.environ['DATA_LOADER_API_URL'] + '/data-loader-api/internal/save'
     else:
-        url = 'http://data-loader-api/data-loader-api/save'
-    headers = {'X-StorageApi-Token': token, 'User-Agent': 'Keboola Sandbox Autosave Request'}
+        url = 'http://data-loader-api/data-loader-api/internal/save'
+    headers = {'Content-Type': 'application/json', 'User-Agent': 'Keboola Sandbox Autosave Request'}
     payload = {'file': {'source': os.path.relpath(file_path), 'tags': ['autosave', 'sandbox-' + sandbox_id] + tags}}
 
     # the timeout is set to > 3min because of the delay on 400 level exception responses
@@ -75,43 +72,30 @@ def saveFile(file_path, sandbox_id, token, log, tags=None):
         raise e
 
 
-def updateApiTimestamp(sandbox_id, token, log):
+def updateApiTimestamp(sandbox_id, log):
     """
-    Update autosave timestamp in Sandboxes API
+    Update autosave timestamp via data-loader-api
     Args:
         sandbox_id: Id of the sandbox
-        token: Keboola Storage token
         log: Logger instance
     """
 
     headers = {
         'Content-Type': 'application/json',
         'User-Agent': 'Keboola Sandbox Autosave Request',
-        'X-StorageApi-Token': token,
     }
-    url = os.environ['SANDBOXES_API_URL'] + '/sandboxes/' + sandbox_id
-    body = json.dumps({'lastAutosaveTimestamp': datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%f')})
-    result = retrySession().patch(url, data=body, headers=headers)
+    if 'DATA_LOADER_API_URL' in os.environ and os.environ['DATA_LOADER_API_URL']:
+        url = 'http://' + os.environ['DATA_LOADER_API_URL'] + '/data-loader-api/internal/activity'
+    else:
+        url = 'http://data-loader-api/data-loader-api/internal/activity'
+
+    result = retrySession().post(url, headers=headers)
     if result.status_code == requests.codes.ok:
         log.info('Successfully saved autosave to Sandboxes API')
     else:
         log.error('Saving autosave to Sandboxes API errored: ' + result.text)
 
 
-def getStorageTokenFromEnv(log):
-    """
-    Find Keboola token in env vars
-    Args:
-        log: Logger instance
-    """
-
-    if 'KBC_TOKEN' in os.environ:
-        return os.environ['KBC_TOKEN']
-    else:
-        log.error('Could not find Keboola Storage API token.')
-        raise Exception('Could not find Keboola Storage API token.')
-
-
 def compressFolder(folder_path):
     """
     Gzip folder
@@ -126,19 +110,18 @@ def compressFolder(folder_path):
     return gz_path
 
 
-def saveFolder(folder_path, sandbox_id, token, log):
+def saveFolder(folder_path, sandbox_id, log):
     """
     Gzip folder and save it to Keboola Storage
     Args:
         folder_path: Path to the folder
         sandbox_id: Id of the sandbox
-        token: Keboola Storage token
         log: Logger instance
     """
     if os.path.exists(folder_path):
         gz_path = compressFolder(folder_path)
         try:
-            saveFile(gz_path, sandbox_id, token, log, ['git'])
+            saveFile(gz_path, sandbox_id, log, ['git'])
         finally:
             if os.path.exists(gz_path):
                 os.remove(gz_path)
@@ -156,10 +139,9 @@ def scriptPostSave(model, os_path, contents_manager, **kwargs):
     log = contents_manager.log
 
     sandbox_id = os.environ['SANDBOX_ID']
-    token = getStorageTokenFromEnv(log)
-    saveFile(os_path, sandbox_id, token, log)
-    updateApiTimestamp(sandbox_id, token, log)
-    saveFolder('/data/.git', sandbox_id, token, log)
+    updateApiTimestamp(sandbox_id, log)
+    saveFile(os_path, sandbox_id, log)
+    saveFolder('/data/.git', sandbox_id, log)
 
 
 def notebookSetup(c):

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name='keboola-sandboxes-notebook-utils',
-    version='1.3.2',
+    version='1.4.0',
     url='https://github.com/keboola/sandboxes-notebook-utils',
     packages=['keboola_notebook_utils'],
     package_dir={'keboola_notebook_utils': ''},

tests/notebookUtils.py

@@ -7,7 +7,7 @@
 import string
 import tempfile
 
-from notebookUtils import compressFolder, getStorageTokenFromEnv, notebookSetup, saveFile, saveFolder, \
+from notebookUtils import compressFolder, notebookSetup, saveFile, saveFolder, \
     scriptPostSave, updateApiTimestamp
 
 def generate_random_string():
@@ -36,12 +36,10 @@ def test_notebookSetup(self):
 
     def test_scriptPostSave(self):
         with requests_mock.Mocker() as m:
-            os.environ['SANDBOXES_API_URL'] = 'http://sandboxes-api'
             os.environ['SANDBOX_ID'] = '123'
             os.environ['DATA_LOADER_API_URL'] = 'dataloader'
-            os.environ['KBC_TOKEN'] = 'token'
-            dataLoaderMock = m.post('http://dataloader/data-loader-api/save', json={'result': 'ok'})
-            apiMock = m.patch('http://sandboxes-api/sandboxes/123', json={'result': 'ok'})
+            dataLoaderMock = m.post('http://dataloader/data-loader-api/internal/save', json={'result': 'ok'})
+            activityMock = m.post('http://dataloader/data-loader-api/internal/activity', json={'result': 'ok'})
 
             contentsManager = type('', (), {})()
             contentsManager.log = logging
@@ -51,35 +49,23 @@ def test_scriptPostSave(self):
             assert 'file' in dataLoaderMock.last_request.text
             assert 'tags' in dataLoaderMock.last_request.text
 
-            assert apiMock.call_count == 1
-            assert 'lastAutosaveTimestamp' in apiMock.last_request.text
-
-    def test_getStorageTokenFromEnvMissing(self):
-        os.environ.pop('KBC_TOKEN')
-        with pytest.raises(Exception):
-            getStorageTokenFromEnv(logging)
-
-    def test_getStorageTokenFromEnvOk(self):
-        token = generate_random_string()
-        os.environ['KBC_TOKEN'] = token
-        assert getStorageTokenFromEnv(logging) == token
+            assert activityMock.call_count == 1
 
     def test_updateApiTimestamp(self):
         with requests_mock.Mocker() as m:
-            os.environ['SANDBOXES_API_URL'] = 'http://sandboxes-api'
-            apiMock = m.patch('http://sandboxes-api/sandboxes/123', json={'result': 'ok'})
+            os.environ['DATA_LOADER_API_URL'] = 'dataloader'
+            activityMock = m.post('http://dataloader/data-loader-api/internal/activity', json={'result': 'ok'})
 
-            updateApiTimestamp('123', 'token', logging)
+            updateApiTimestamp('123', logging)
 
-            assert apiMock.call_count == 1
-            assert 'lastAutosaveTimestamp' in apiMock.last_request.text
+            assert activityMock.call_count == 1
 
     def test_saveFile(self):
         with requests_mock.Mocker() as m:
             os.environ['DATA_LOADER_API_URL'] = 'dataloader'
-            dataLoaderMock = m.post('http://dataloader/data-loader-api/save', json={'result': 'ok'})
+            dataLoaderMock = m.post('http://dataloader/data-loader-api/internal/save', json={'result': 'ok'})
 
-            saveFile('/file/path', '123', 'token', logging)
+            saveFile('/file/path', '123', logging)
 
             assert dataLoaderMock.call_count == 1
             response = json.loads(dataLoaderMock.last_request.text)
@@ -106,14 +92,14 @@ def test_compressFolder(self):
     def test_saveFolder(self):
         with requests_mock.Mocker() as m:
             os.environ['DATA_LOADER_API_URL'] = 'dataloader'
-            dataLoaderMock = m.post('http://dataloader/data-loader-api/save', json={'result': 'ok'})
+            dataLoaderMock = m.post('http://dataloader/data-loader-api/internal/save', json={'result': 'ok'})
 
             folder_prepare = tempfile.mkdtemp() + '/.git'
             os.mkdir(folder_prepare)
             f = open(folder_prepare + '/file.txt', 'a')
             f.write('content')
             f.close()
-            saveFolder(folder_prepare, '123', 'token', logging)
+            saveFolder(folder_prepare, '123', logging)
 
             assert dataLoaderMock.call_count == 1
             response = json.loads(dataLoaderMock.last_request.text)