From 5fa577780157b83fa8319a407c1e52c3e598d88a Mon Sep 17 00:00:00 2001 From: Wannes Rombouts Date: Fri, 26 Jun 2015 17:26:08 +0200 Subject: [PATCH 1/6] Draft implementation of python server. --- client/bush/api.py | 39 ++++++++----------- client/bush/cli.py | 10 ++++- client/bush/server.py | 84 +++++++++++++++++++++++++++++++++++++++++ client/requirements.txt | 3 ++ 4 files changed, 112 insertions(+), 24 deletions(-) create mode 100644 client/bush/server.py diff --git a/client/bush/api.py b/client/bush/api.py index 32462a6..d7303d6 100644 --- a/client/bush/api.py +++ b/client/bush/api.py @@ -22,20 +22,18 @@ class BushFile(): - def __init__(self, tag, name, date=0, compressed=None, url=None, **kwargs): - - if compressed is None: - compressed = name.endswith('.tar.gz') + def __init__(self, tag, name, date=0, url=None, **kwargs): + if name.endswith('.tar.gz'): + name = name[:-7] self.tag = tag - self.compressed = compressed - self.name = name[:-7] if self.compressed else name + self.name = name self.date = arrow.get(date) self.url = url def __repr__(self): - return "BushFile(tag=%s, name=%s, date=%s, compressed=%s)" % ( - self.tag, self.name, self.data, self.compressed) + return "BushFile(tag=%s, name=%s, date=%s, url=%s)" % ( + self.tag, self.name, self.data, self.url) def output(self, file=sys.stdout, align=0, extended=False): if not extended: @@ -74,8 +72,9 @@ def confirmation(self, msg, level): if level > INFO: raise RuntimeError(msg) - def url(self, url): - return urllib.parse.urljoin(self.base, url) + def url(self, path, *args): + args = (urllib.parse.quote(arg, safe='') for arg in args) + return urllib.parse.urljoin(self.base, path.format(*args)) def tag_for_path(self, filepath): basename = os.path.basename(filepath) @@ -124,10 +123,9 @@ def check_target(self, dest, fdest, isdir=False, placeholder=True): return True def list(self): - r = self.requests.get(self.url("index.php?request=list")) + r = self.requests.get(self.url("/files/")) self.assert_response(r) - return [BushFile(url=self.getddl(f['tag']), **f) - for f in json.loads(r.text)] + return [BushFile(k, **v) for k, v in r.json().items()] def upload(self, filepath, tag=None, callback=None): @@ -142,7 +140,6 @@ def upload(self, filepath, tag=None, callback=None): raise ValueError("Must specify tag for multifile.") tag = tag or self.tag_for_path(filepaths[0]) - tmp = tempfile.TemporaryFile() tar = tarfile.open("bush_upload.tar.gz", "w:gz", fileobj=tmp) @@ -158,7 +155,6 @@ def upload(self, filepath, tag=None, callback=None): filename = "%s.tar.gz" % ", ".join(basenames) encoder = MultipartEncoder(fields={ - 'tag': tag, 'file': (filename, tmp, 'application/octet-stream') }) @@ -172,8 +168,7 @@ def _callback(monitor): _callback = None monitor = MultipartEncoderMonitor(encoder, _callback) - - r = self.requests.post(self.url('index.php?request=upload'), data=monitor, + r = self.requests.put(self.url('/files/{}', tag), data=monitor, headers={'Content-Type': monitor.content_type}) if _callback: @@ -187,12 +182,11 @@ def _callback(monitor): def getddl(self, tag): tag = urllib.parse.quote(tag) - return self.url("index.php?request=get&tag=%s" % tag) + return self.url("/files/{}", tag) def download(self, tag, dest, callback=None, chunksz=8192): - r = self.requests.get(self.url("index.php?request=get"), - params={"tag": tag}, stream=True) + r = self.requests.get(self.url("/files/{}", tag), stream=True) self.assert_response(r) @@ -301,8 +295,7 @@ def check_member(member): def delete(self, tag): - r = self.requests.get(self.url("index.php?request=delete"), - params={"tag": tag}) + r = self.requests.delete(self.url("/files/{}", tag)) self.assert_response(r) data = r.json() @@ -310,7 +303,7 @@ def delete(self, tag): def reset(self): - r = self.requests.get(self.url("index.php?request=reset")) + r = self.requests.delete(self.url("/files/")) self.assert_response(r) diff --git a/client/bush/cli.py b/client/bush/cli.py index 26cdbd5..fc31275 100644 --- a/client/bush/cli.py +++ b/client/bush/cli.py @@ -14,7 +14,6 @@ import bush.api import bush.config - class ShowProgress(): def __init__(self, total): @@ -106,6 +105,12 @@ def do_reset(api, args): api.reset() +def do_serve(api, args): + + from bush.server import app + app.run(debug=True) + + def main(): parser = argparse.ArgumentParser(description="Simplistic file sharing. ", @@ -149,6 +154,9 @@ def main(): sub = subs.add_parser('reset', help="delete all files") sub.set_defaults(callback=do_reset) + sub = subs.add_parser('serve', help="act as a bush server") + sub.set_defaults(callback=do_serve) + parser.add_argument('-u', '--url', help="API endpoint") parser.add_argument('-U', '--username', help="API username") parser.add_argument('-P', '--password', help="API password") diff --git a/client/bush/server.py b/client/bush/server.py new file mode 100644 index 0000000..d08b8f9 --- /dev/null +++ b/client/bush/server.py @@ -0,0 +1,84 @@ +import os +import uuid +import sqlite3 +import os.path +import datetime + +from flask import Flask, request, redirect, send_file +from flask_restful import Resource, Api, abort +from flask.ext import shelve + +app = Flask(__name__) + +app.config['UPLOAD_FOLDER'] = os.path.realpath('./data/') +app.config['SHELVE_FILENAME'] = os.path.realpath('./data/files.db') + +api = Api(app) +shelve.init_app(app) + +class FileList(Resource): + + def get(self): + db = shelve.get_shelve('c') + return {k: {'name': v['name'], + 'date': v['date'].isoformat(), + 'url': api.url_for(File, tag=k, _external=True)} + for k, v in db.items()} + + def delete(self): + db = shelve.get_shelve('c') + for f in db.values(): + try: + os.remove(f['path']) + except FileNotFoundError: + pass + db.clear() + return {} + + +class File(Resource): + + def get(self, tag): + db = shelve.get_shelve('c') + + try: + f = db[tag] + except KeyError: + abort(404) + + return send_file(f['path'], as_attachment=True, attachment_filename=f['name']) + + def put(self, tag): + path = "%s/%s" % (app.config['UPLOAD_FOLDER'], uuid.uuid4()) + + f = request.files['file'] + f.save(path) + + db = shelve.get_shelve('c') + db[tag] = { + 'name': f.filename, + 'path': path, + 'date': datetime.datetime.now() + } + + return {}, 201 + + def delete(self, tag): + db = shelve.get_shelve('c') + + try: + f = db.pop(tag) + except KeyError: + abort(404) + + try: + os.remove(f['path']) + except FileNotFoundError: + pass + + return {}, 200 + + # TODO: also delete file on FS. + +api.add_resource(FileList, '/files/') +api.add_resource(File, '/files/') diff --git a/client/requirements.txt b/client/requirements.txt index feab848..40083e2 100644 --- a/client/requirements.txt +++ b/client/requirements.txt @@ -3,3 +3,6 @@ pyyaml appdirs progressbar2 requests-toolbelt +flask +flask-restful +flask-shelve From 65be91f6fbdbde7e85c7e97f64b5383c95960a13 Mon Sep 17 00:00:00 2001 From: Wannes Rombouts Date: Fri, 26 Jun 2015 17:57:29 +0200 Subject: [PATCH 2/6] Use arrow instead of datetime. This handles timezones out of the box. --- client/bush/server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/bush/server.py b/client/bush/server.py index d08b8f9..86d51fb 100644 --- a/client/bush/server.py +++ b/client/bush/server.py @@ -2,7 +2,8 @@ import uuid import sqlite3 import os.path -import datetime + +import arrow from flask import Flask, request, redirect, send_file from flask_restful import Resource, Api, abort @@ -58,7 +59,7 @@ def put(self, tag): db[tag] = { 'name': f.filename, 'path': path, - 'date': datetime.datetime.now() + 'date': arrow.now() } return {}, 201 From bf9c710900e72f8d6a78829d81f9c044d528e541 Mon Sep 17 00:00:00 2001 From: Wannes Rombouts Date: Fri, 3 Jul 2015 14:56:02 +0200 Subject: [PATCH 3/6] Remove PHP server because it is incompatible. --- server/README.md | 23 --------- server/data/files.sqlite | 0 server/index.php | 106 --------------------------------------- server/storage.php | 79 ----------------------------- tests/config/config.yaml | 4 +- 5 files changed, 2 insertions(+), 210 deletions(-) delete mode 100644 server/README.md delete mode 100644 server/data/files.sqlite delete mode 100644 server/index.php delete mode 100644 server/storage.php diff --git a/server/README.md b/server/README.md deleted file mode 100644 index f9f786e..0000000 --- a/server/README.md +++ /dev/null @@ -1,23 +0,0 @@ - -## get started - -```sh -cd server -php -S 127.0.0.1:8080 -``` - -## requirements - - - `php5` (5.3 for normal use, but 5.4 is required to use PHP built-in server as is shown in the get started) - - `php5-sqlite` - -## authentication & tips - -#### By default all files are public. - -Please don't use `php -S` for production, the built-in server must be used for developments / tests only ([php-doc-webserver](https://php.net/manual/features.commandline.webserver.php)). - -Use apache, nginx or any other web servers. - -It is __strongly reccomended__ that you add a Basic Authentication on the bush root directory. -The client supports this and it is the only option if you don't want to share all your files publicly. diff --git a/server/data/files.sqlite b/server/data/files.sqlite deleted file mode 100644 index e69de29..0000000 diff --git a/server/index.php b/server/index.php deleted file mode 100644 index 80cdfbe..0000000 --- a/server/index.php +++ /dev/null @@ -1,106 +0,0 @@ -addFile($_POST['tag'], $_FILES['file']['name'], 0); - if ($id !== NULL) { - $filepath = DATAPATH."/".$id.".bin"; - if (move_uploaded_file($_FILES['file']['tmp_name'], $filepath) && file_exists($filepath)) { - header('HTTP/1.1 201 Created'); - echo json_encode(array( - 'status' => 'OK', - 'tag' => $_POST['tag'] - )); - exit(); - } - } - } - header('HTTP/1.1 400 Bad Request'); - echo json_encode(array( - 'status' => 'KO', - 'message' => 'Upload requested, but no uploaded file found.') - ); - } - - if ($_GET['request'] == "list") { - $files = $db->getAllFiles(); - foreach ($files as &$file) { - $filepath = DATAPATH."/".$file['id'].".bin"; - $file['date'] = date('c', filemtime($filepath)); - } - echo json_encode($files); - exit(); - } - - if ($_GET['request'] == 'get' && !empty($_GET['tag'])) { - $file = $db->getFile($_GET['tag']); - if (!empty($file['id']) && !empty($file['name'])) { - $filepath = DATAPATH."/".$file['id'].".bin"; - ob_end_flush(); - header("Content-Disposition: attachment; filename=" . urlencode($file['name'])); - header("Content-Type: application/force-download"); - header("Content-Type: application/octet-stream"); - header("Content-Type: application/download"); - header("Content-Description: File Transfer"); - header("Content-Length: " . filesize($filepath)); - readfile($filepath); - } - else { - header('HTTP/1.1 404 Not Found'); - echo json_encode(array( - 'status' => 'KO', - 'message'=> 'Non-existing tag or filename') - ); - } - exit(); - } - - if ($_GET['request'] == 'delete' && !empty($_GET['tag'])) { - $ret = $db->getFile($_GET['tag']); - if (!empty($ret)) { - $db->deleteFileTag($_GET['tag']); - echo json_encode(array( - 'status' => 'OK' - )); - } - else { - header('HTTP/1.1 404 Not Found'); - echo json_encode(array( - 'status' => 'KO', - 'message'=> 'Non-existing tag or filename') - ); - } - exit(); - } - - if ($_GET['request'] == 'reset') { - $files = $db->getAllFiles(); - $i = 0; - foreach ($files as $file) { - $db->deleteFileId($file['id']); - ++$i; - } - echo json_encode(array( - 'status' => 'OK', - 'files_deleted' => $i - )); - exit(); - } -} - -header('HTTP/1.1 400 Bad Request'); -echo json_encode(array( - 'status' => 'KO', - 'message' => 'Invalid request') -); diff --git a/server/storage.php b/server/storage.php deleted file mode 100644 index b9d5125..0000000 --- a/server/storage.php +++ /dev/null @@ -1,79 +0,0 @@ -_dataPath = $dataPath; - - $this->_db = new PDO('sqlite:'.$sqliteFile); - - $this->_db->exec("CREATE TABLE IF NOT EXISTS files ( - id INTEGER PRIMARY KEY, - name TEXT, - tag TEXT, - compressed INTEGER)"); - } - - public function getFile($tag) { - $sql = "SELECT id, name FROM files WHERE tag = :tag"; - $stmt = $this->_db->prepare($sql); - $stmt->bindParam(':tag', $tag); - $stmt->execute(); - - $res = $stmt->fetch(PDO::FETCH_ASSOC); - - if ($res) { - return $res; - } - - $sql = "SELECT id, name FROM files WHERE name = :name"; - $stmt = $this->_db->prepare($sql); - $stmt->bindParam(':name', $tag); - $stmt->execute(); - - $res = $stmt->fetch(PDO::FETCH_ASSOC); - - return $res; - } - - public function getAllFiles() { - $sql = "SELECT id, name, tag FROM files"; - $stmt = $this->_db->prepare($sql); - $stmt->execute(); - - $res = $stmt->fetchAll(PDO::FETCH_ASSOC); - return $res; - } - - public function deleteFileId($id) { - $sql = "DELETE FROM files WHERE id = :id"; - $stmt = $this->_db->prepare($sql); - $stmt->bindParam(':id', $id); - $stmt->execute(); - @unlink($this->_dataPath.'/'.$id.'.bin'); - } - - public function deleteFileTag($tag) { - $file = $this->getFile($tag); - if ($file) { - $this->deleteFileId($file['id']); - } - } - - public function addFile($tag, $name, $compressed) { - $file = $this->getFile($tag); - if ($file) { - $this->deleteFileId($file['id']); - } - $sql = "INSERT INTO files (name, tag, compressed) VALUES (:name, :tag, :compressed)"; - $stmt = $this->_db->prepare($sql); - $stmt->bindParam(':tag', $tag); - $stmt->bindParam(':name', $name); - $stmt->bindParam(':compressed', $compressed); - $stmt->execute(); - return $this->_db->lastInsertId(); - } -} diff --git a/tests/config/config.yaml b/tests/config/config.yaml index 55803f1..0963b33 100644 --- a/tests/config/config.yaml +++ b/tests/config/config.yaml @@ -1,4 +1,4 @@ servers: - - url: http://127.0.0.1:8080/ - alias: bestpig + - url: http://127.0.0.1:5000/ + alias: local From 4c80e09ac074087f21f44e70801b55f060bd2d15 Mon Sep 17 00:00:00 2001 From: Wannes Rombouts Date: Mon, 6 Jul 2015 11:17:49 +0200 Subject: [PATCH 4/6] Use specified url when serving. --- client/bush/api.py | 11 ++++++----- client/bush/cli.py | 29 +++++++++++++++++------------ client/bush/server.py | 3 +-- client/setup.py | 2 +- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/client/bush/api.py b/client/bush/api.py index d7303d6..e0effdf 100644 --- a/client/bush/api.py +++ b/client/bush/api.py @@ -52,7 +52,8 @@ def output(self, file=sys.stdout, align=0, extended=False): class BushAPI(): def __init__(self, base, username=None, password=None): - self.base = base + self.base_url = base + self.base = urllib.parse.urlparse(self.base_url) self.requests = requests.session() @@ -60,13 +61,13 @@ def __init__(self, base, username=None, password=None): 'User-Agent': 'bush.py.%s' % bush.meta.__version__, }) - scheme = urllib.parse.urlparse(self.base)[0] - if (username or password) and scheme != 'https': + if (username or password) and self.base.scheme != 'https': if not self.confirmation("Sending credentials over %r is insecure." - % scheme, level=EXTREME): + % self.base.scheme, level=EXTREME): raise KeyboardInterrupt() self.requests.auth = (username, password) + def confirmation(self, msg, level): # Don't confirm anything by default! if level > INFO: @@ -74,7 +75,7 @@ def confirmation(self, msg, level): def url(self, path, *args): args = (urllib.parse.quote(arg, safe='') for arg in args) - return urllib.parse.urljoin(self.base, path.format(*args)) + return urllib.parse.urljoin(self.base_url, path.format(*args)) def tag_for_path(self, filepath): basename = os.path.basename(filepath) diff --git a/client/bush/cli.py b/client/bush/cli.py index fc31275..318490c 100644 --- a/client/bush/cli.py +++ b/client/bush/cli.py @@ -1,15 +1,13 @@ import os import sys -import time -import pprint import argparse -import progressbar import datetime -import operator +import contextlib from distutils import util import arrow +import progressbar import bush.api import bush.config @@ -48,6 +46,15 @@ def confirmation(self, msg, level): return status +@contextlib.contextmanager +def user_friendly_errors(exceptions=Exception, debug=False): + try: + yield + except KeyboardInterrupt: + print('interrupted', file=sys.stderr) + except () if debug else exceptions as e: + exit(e) + def do_list(api, args): files = api.list() @@ -106,9 +113,8 @@ def do_reset(api, args): def do_serve(api, args): - from bush.server import app - app.run(debug=True) + app.run(host=api.base.hostname, port=api.base.port, debug=args.debug) def main(): @@ -169,6 +175,9 @@ def main(): config = bush.config.load_config(args.config) + if args.callback is do_serve and args.url is None: + args.url = 'local' + servers = config.get('servers') or [{}] for server in servers: @@ -190,11 +199,7 @@ def main(): username = args.username or server.get('username') password = args.password or server.get('password') + api = UIAPI(url, username=username, password=password) - try: - api = UIAPI(url, username=username, password=password) + with user_friendly_errors(debug=args.debug): args.callback(api, args) - except KeyboardInterrupt: - print('interrupted') # Canceled by user :( - except Exception if not args.debug else () as e: - exit(e) diff --git a/client/bush/server.py b/client/bush/server.py index 86d51fb..01f1909 100644 --- a/client/bush/server.py +++ b/client/bush/server.py @@ -1,6 +1,5 @@ import os import uuid -import sqlite3 import os.path import arrow @@ -23,7 +22,7 @@ def get(self): db = shelve.get_shelve('c') return {k: {'name': v['name'], 'date': v['date'].isoformat(), - 'url': api.url_for(File, tag=k, _external=True)} + 'url': api.url_for(File, tag=k)} for k, v in db.items()} def delete(self): diff --git a/client/setup.py b/client/setup.py index b103a60..a9df1a0 100644 --- a/client/setup.py +++ b/client/setup.py @@ -5,7 +5,7 @@ if sys.version_info < (3, 3, 0): from datetime import datetime - sys.stdout.write("It's %d. This requires Python > 3.3.\n" + sys.stdout.write("It's %d. This requires Python >= 3.3.\n" % datetime.now().year) sys.exit(1) From 4e67a40215423c8192dd4c3f0d0752979ae178a7 Mon Sep 17 00:00:00 2001 From: Wannes Rombouts Date: Mon, 6 Jul 2015 12:01:33 +0200 Subject: [PATCH 5/6] Get datadir from configuration or arguments. --- client/bush/cli.py | 23 +++++++++++++---------- client/bush/server.py | 10 +++++++--- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/client/bush/cli.py b/client/bush/cli.py index 318490c..c6df8e7 100644 --- a/client/bush/cli.py +++ b/client/bush/cli.py @@ -56,14 +56,14 @@ def user_friendly_errors(exceptions=Exception, debug=False): exit(e) -def do_list(api, args): +def do_list(api, args, config): files = api.list() maxlen = max(len(f.tag) for f in files) if files else 0 for f in files: f.output(align=maxlen, extended=args.exact) -def do_wait(api, args): +def do_wait(api, args, config): latest = None update = arrow.now() - datetime.timedelta(seconds=args.age) @@ -84,7 +84,7 @@ def do_wait(api, args): api.download(latest.tag, args.dest, callback=ShowProgress) -def do_upload(api, args): +def do_upload(api, args, config): if args.tag is not None: tag = args.tag @@ -100,21 +100,22 @@ def do_upload(api, args): api.upload(args.file, tag=tag, callback=ShowProgress) -def do_download(api, args): +def do_download(api, args, config): api.download(args.tag, args.dest, callback=ShowProgress) -def do_delete(api, args): +def do_delete(api, args, config): api.delete(args.tag) -def do_reset(api, args): +def do_reset(api, args, config): api.reset() -def do_serve(api, args): - from bush.server import app - app.run(host=api.base.hostname, port=api.base.port, debug=args.debug) +def do_serve(api, args, config): + from bush import server + server.config_datadir(args.datadir or config.get('datadir', './data/')) + server.app.run(host=api.base.hostname, port=api.base.port, debug=args.debug) def main(): @@ -162,6 +163,8 @@ def main(): sub = subs.add_parser('serve', help="act as a bush server") sub.set_defaults(callback=do_serve) + sub.add_argument('-d', '--datadir', + help="path do the directory used for storing files") parser.add_argument('-u', '--url', help="API endpoint") parser.add_argument('-U', '--username', help="API username") @@ -202,4 +205,4 @@ def main(): api = UIAPI(url, username=username, password=password) with user_friendly_errors(debug=args.debug): - args.callback(api, args) + args.callback(api, args, server) diff --git a/client/bush/server.py b/client/bush/server.py index 01f1909..ab4b198 100644 --- a/client/bush/server.py +++ b/client/bush/server.py @@ -10,8 +10,13 @@ app = Flask(__name__) -app.config['UPLOAD_FOLDER'] = os.path.realpath('./data/') -app.config['SHELVE_FILENAME'] = os.path.realpath('./data/files.db') +def config_datadir(datadir): + app.config.update( + UPLOAD_FOLDER=os.path.realpath(datadir), + SHELVE_FILENAME=os.path.realpath(os.path.join(datadir, 'files.db')) + ) + +config_datadir('./data/') api = Api(app) shelve.init_app(app) @@ -78,7 +83,6 @@ def delete(self, tag): return {}, 200 - # TODO: also delete file on FS. api.add_resource(FileList, '/files/') api.add_resource(File, '/files/') From 63c98b8fd9bb020e972a8a64779ebba9586fb138 Mon Sep 17 00:00:00 2001 From: Wannes Rombouts Date: Mon, 6 Jul 2015 14:05:02 +0200 Subject: [PATCH 6/6] Show absolute url in file info. --- client/bush/api.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/client/bush/api.py b/client/bush/api.py index e0effdf..1b90fdf 100644 --- a/client/bush/api.py +++ b/client/bush/api.py @@ -22,10 +22,13 @@ class BushFile(): - def __init__(self, tag, name, date=0, url=None, **kwargs): + def __init__(self, tag, name, date=0, url=None, base=None, **kwargs): if name.endswith('.tar.gz'): name = name[:-7] + if base is not None: + url = urllib.parse.urljoin(base, url) + self.tag = tag self.name = name self.date = arrow.get(date) @@ -126,7 +129,8 @@ def check_target(self, dest, fdest, isdir=False, placeholder=True): def list(self): r = self.requests.get(self.url("/files/")) self.assert_response(r) - return [BushFile(k, **v) for k, v in r.json().items()] + return [BushFile(k, base=self.base_url, **v) + for k, v in r.json().items()] def upload(self, filepath, tag=None, callback=None):