AJDA-2169 - Switch autosave to data-loader internal endpoint

.github/workflows/test.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ['3.10']
 
     steps:
     - uses: actions/checkout@v2

notebookUtils.py

@@ -37,14 +37,13 @@ def retrySession(
     return session
 
 
-def saveFile(file_path, sandbox_id, token, log, tags=None):
+def saveFile(file_path, sandbox_id, log, tags=None):
     """
     Construct a requests POST call with args and kwargs and process the
     results.
     Args:
         file_path: The relative path to the file from the datadir, including filename and extension
         sandbox_id: Id of the sandbox
-        token: Keboola Storage token
         log: Logger instance
         tags: Additional tags for the file
     Returns:
@@ -57,10 +56,10 @@ def saveFile(file_path, sandbox_id, token, log, tags=None):
     if tags is None:
         tags = []
     if 'DATA_LOADER_API_URL' in os.environ and os.environ['DATA_LOADER_API_URL']:
-        url = 'http://' + os.environ['DATA_LOADER_API_URL'] + '/data-loader-api/save'
+        url = 'http://' + os.environ['DATA_LOADER_API_URL'] + '/data-loader-api/internal/save'
     else:
-        url = 'http://data-loader-api/data-loader-api/save'
-    headers = {'X-StorageApi-Token': token, 'User-Agent': 'Keboola Sandbox Autosave Request'}
+        url = 'http://data-loader-api/data-loader-api/internal/save'
+    headers = {'Content-Type': 'application/json', 'User-Agent': 'Keboola Sandbox Autosave Request'}
     payload = {'file': {'source': os.path.relpath(file_path), 'tags': ['autosave', 'sandbox-' + sandbox_id] + tags}}
 
     # the timeout is set to > 3min because of the delay on 400 level exception responses
@@ -98,20 +97,6 @@ def updateApiTimestamp(sandbox_id, log):
         log.error('Saving autosave to Sandboxes API errored: ' + result.text)
 
 
-def getStorageTokenFromEnv(log):
-    """
-    Find Keboola token in env vars
-    Args:
-        log: Logger instance
-    """
-
-    if 'KBC_TOKEN' in os.environ:
-        return os.environ['KBC_TOKEN']
-    else:
-        log.error('Could not find Keboola Storage API token.')
-        raise Exception('Could not find Keboola Storage API token.')
-
-
 def compressFolder(folder_path):
     """
     Gzip folder
@@ -126,19 +111,18 @@ def compressFolder(folder_path):
     return gz_path
 
 
-def saveFolder(folder_path, sandbox_id, token, log):
+def saveFolder(folder_path, sandbox_id, log):
     """
     Gzip folder and save it to Keboola Storage
     Args:
         folder_path: Path to the folder
         sandbox_id: Id of the sandbox
-        token: Keboola Storage token
         log: Logger instance
     """
     if os.path.exists(folder_path):
         gz_path = compressFolder(folder_path)
         try:
-            saveFile(gz_path, sandbox_id, token, log, ['git'])
+            saveFile(gz_path, sandbox_id, log, ['git'])
         finally:
             if os.path.exists(gz_path):
                 os.remove(gz_path)
@@ -156,13 +140,12 @@ def scriptPostSave(model, os_path, contents_manager, **kwargs):
     log = contents_manager.log
 
     sandbox_id = os.environ['SANDBOX_ID']
-    token = getStorageTokenFromEnv(log)
     updateApiTimestamp(sandbox_id, log)
 
     has_persistent_storage = os.getenv('HAS_PERSISTENT_STORAGE', 'False').lower() in ('true', '1')
     if not has_persistent_storage:
-        saveFile(os_path, sandbox_id, token, log)
-        saveFolder('/data/.git', sandbox_id, token, log)
+        saveFile(os_path, sandbox_id, log)
+        saveFolder('/data/.git', sandbox_id, log)
 
 
 def notebookSetup(c):

requirements.txt

@@ -24,7 +24,7 @@ pycodestyle==2.6.0
 pyflakes==2.2.0
 Pygments==2.7.4
 pyparsing==2.4.7
-pytest==5.4.3
+pytest==6.2.5
 requests==2.25.1
 requests-mock==1.8.0
 six==1.15.0

setup.py

@@ -2,9 +2,10 @@
 
 setup(
     name='keboola-sandboxes-notebook-utils',
-    version='2.2.0',
+    version='2.3.0',
     url='https://github.com/keboola/sandboxes-notebook-utils',
     packages=['keboola_notebook_utils'],
     package_dir={'keboola_notebook_utils': ''},
-    requires=['pip']
+    python_requires='>=3.10',
+    install_requires=['requests'],
 )

tests/notebookUtils.py

@@ -7,7 +7,7 @@
 import string
 import tempfile
 
-from notebookUtils import compressFolder, getStorageTokenFromEnv, notebookSetup, saveFile, saveFolder, \
+from notebookUtils import compressFolder, notebookSetup, saveFile, saveFolder, \
     scriptPostSave, updateApiTimestamp
 
 def generate_random_string():
@@ -41,11 +41,10 @@ def test_scriptPostSave(self):
         with requests_mock.Mocker() as m:
             os.environ['SANDBOX_ID'] = '123'
             os.environ['DATA_LOADER_API_URL'] = 'dataloader'
-            os.environ['KBC_TOKEN'] = 'token'
             if 'HAS_PERSISTENT_STORAGE' in os.environ:
                 del os.environ['HAS_PERSISTENT_STORAGE']
 
-            dataLoaderMock = m.post('http://dataloader/data-loader-api/save', json={'result': 'ok'})
+            dataLoaderMock = m.post('http://dataloader/data-loader-api/internal/save', json={'result': 'ok'})
             dataLoaderActivityMock = m.post('http://dataloader/data-loader-api/internal/activity', json={'result': 'ok'})
 
             contentsManager = type('', (), {})()
@@ -62,9 +61,8 @@ def test_scriptPostSave_disabledPersistentStorage(self):
         with requests_mock.Mocker() as m:
             os.environ['SANDBOX_ID'] = '123'
             os.environ['DATA_LOADER_API_URL'] = 'dataloader'
-            os.environ['KBC_TOKEN'] = 'token'
             os.environ['HAS_PERSISTENT_STORAGE'] = '0'
-            dataLoaderMock = m.post('http://dataloader/data-loader-api/save', json={'result': 'ok'})
+            dataLoaderMock = m.post('http://dataloader/data-loader-api/internal/save', json={'result': 'ok'})
             dataLoaderActivityMock = m.post('http://dataloader/data-loader-api/internal/activity', json={'result': 'ok'})
 
             contentsManager = type('', (), {})()
@@ -77,14 +75,12 @@ def test_scriptPostSave_disabledPersistentStorage(self):
 
             assert dataLoaderActivityMock.call_count == 1
 
-
     def test_scriptPostSave_enabledPersistentStorage(self):
         with requests_mock.Mocker() as m:
             os.environ['SANDBOX_ID'] = '123'
             os.environ['DATA_LOADER_API_URL'] = 'dataloader'
-            os.environ['KBC_TOKEN'] = 'token'
             os.environ['HAS_PERSISTENT_STORAGE'] = '1'
-            dataLoaderMock = m.post('http://dataloader/data-loader-api/save', json={'result': 'ok'})
+            dataLoaderMock = m.post('http://dataloader/data-loader-api/internal/save', json={'result': 'ok'})
             dataLoaderActivityMock = m.post('http://dataloader/data-loader-api/internal/activity', json={'result': 'ok'})
 
             contentsManager = type('', (), {})()
@@ -94,19 +90,11 @@ def test_scriptPostSave_enabledPersistentStorage(self):
             assert dataLoaderMock.call_count == 0
             assert dataLoaderActivityMock.call_count == 1
 
-    def test_getStorageTokenFromEnvMissing(self):
-        os.environ.pop('KBC_TOKEN')
-        with pytest.raises(Exception):
-            getStorageTokenFromEnv(logging)
-
-    def test_getStorageTokenFromEnvOk(self):
-        token = generate_random_string()
-        os.environ['KBC_TOKEN'] = token
-        assert getStorageTokenFromEnv(logging) == token
-
     def test_updateApiTimestamp(self):
         with requests_mock.Mocker() as m:
-            dataLoaderActivityMock = m.post('http://dataloader/data-loader-api/internal/activity', json={'result': 'ok'})
+            os.environ['DATA_LOADER_API_URL'] = 'dataloader'
+            url = 'http://dataloader/data-loader-api/internal/activity'
+            dataLoaderActivityMock = m.post(url, json={'result': 'ok'})
 
             updateApiTimestamp('123', logging)
 
@@ -115,9 +103,9 @@ def test_updateApiTimestamp(self):
     def test_saveFile(self):
         with requests_mock.Mocker() as m:
             os.environ['DATA_LOADER_API_URL'] = 'dataloader'
-            dataLoaderMock = m.post('http://dataloader/data-loader-api/save', json={'result': 'ok'})
+            dataLoaderMock = m.post('http://dataloader/data-loader-api/internal/save', json={'result': 'ok'})
 
-            saveFile('/file/path', '123', 'token', logging)
+            saveFile('/file/path', '123', logging)
 
             assert dataLoaderMock.call_count == 1
             response = json.loads(dataLoaderMock.last_request.text)
@@ -144,14 +132,14 @@ def test_compressFolder(self):
     def test_saveFolder(self):
         with requests_mock.Mocker() as m:
             os.environ['DATA_LOADER_API_URL'] = 'dataloader'
-            dataLoaderMock = m.post('http://dataloader/data-loader-api/save', json={'result': 'ok'})
+            dataLoaderMock = m.post('http://dataloader/data-loader-api/internal/save', json={'result': 'ok'})
 
             folder_prepare = tempfile.mkdtemp() + '/.git'
             os.mkdir(folder_prepare)
             f = open(folder_prepare + '/file.txt', 'a')
             f.write('content')
             f.close()
-            saveFolder(folder_prepare, '123', 'token', logging)
+            saveFolder(folder_prepare, '123', logging)
 
             assert dataLoaderMock.call_count == 1
             response = json.loads(dataLoaderMock.last_request.text)