diff --git a/.gitignore b/.gitignore index 0d20b64..a7a425b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ +.DS_Store +config-untracked.cfg *.pyc diff --git a/CreateBag.workflow/Contents/Info.plist b/CreateBag.workflow/Contents/Info.plist new file mode 100644 index 0000000..cc32396 --- /dev/null +++ b/CreateBag.workflow/Contents/Info.plist @@ -0,0 +1,27 @@ + + + + + NSServices + + + NSMenuItem + + default + Transfer to SFU Archives + + NSMessage + runWorkflowAsService + NSRequiredContext + + NSApplicationIdentifier + com.apple.finder + + NSSendFileTypes + + public.folder + + + + + diff --git a/CreateBag.workflow/Contents/QuickLook/Thumbnail.png b/CreateBag.workflow/Contents/QuickLook/Thumbnail.png new file mode 100644 index 0000000..d5cd3d1 Binary files /dev/null and b/CreateBag.workflow/Contents/QuickLook/Thumbnail.png differ diff --git a/CreateBag.workflow/Contents/document.wflow b/CreateBag.workflow/Contents/document.wflow new file mode 100644 index 0000000..348cbf0 --- /dev/null +++ b/CreateBag.workflow/Contents/document.wflow @@ -0,0 +1,271 @@ + + + + + AMApplicationBuild + 247.1 + AMApplicationVersion + 2.1.1 + AMDocumentVersion + 2 + actions + + + action + + AMAccepts + + Container + List + Optional + + Types + + com.apple.cocoa.string + + + AMActionVersion + 2.0.1 + AMParameterProperties + + COMMAND_STRING + + CheckedForUserDefaultShell + + inputMethod + + shell + + source + + + AMProvides + + Container + List + Types + + com.apple.cocoa.string + + + ActionBundlePath + /System/Library/Automator/Run Shell Script.action + ActionName + Run Shell Script + ActionParameters + + COMMAND_STRING + IFS=$'\n'; for f in "$@"; do cd "`dirname "$f"`"; ~/.createbag/createbag $f; done + CheckedForUserDefaultShell + + inputMethod + 1 + shell + /bin/bash + source + + + Application + + Automator + + BundleIdentifier + com.apple.RunShellScript + CFBundleVersion + 2.0.1 + CanShowSelectedItemsWhenRun + + CanShowWhenRun + + Category + + AMCategoryUtilities + + Class Name + RunShellScriptAction + InputUUID + 557601C5-E261-4E7F-83D2-D75A2368F3D8 + Keywords + + Shell + Script + Command + Run + Unix + + OutputUUID + 433CCE7D-C69F-48B9-B746-78E1AE17286B + UUID + EDBB2CA0-D5A3-4B34-A473-3700B5EE75A5 + UnlocalizedApplications + + Automator + + arguments + + 0 + + default value + 0 + name + inputMethod + required + 0 + type + 0 + uuid + 0 + + 1 + + default value + + name + source + required + 0 + type + 0 + uuid + 1 + + 2 + + default value + + name + CheckedForUserDefaultShell + required + 0 + type + 0 + uuid + 2 + + 3 + + default value + + name + COMMAND_STRING + required + 0 + type + 0 + uuid + 3 + + 4 + + default value + /bin/sh + name + shell + required + 0 + type + 0 + uuid + 4 + + + isViewVisible + + location + 309.500000:554.000000 + nibPath + /System/Library/Automator/Run Shell Script.action/Contents/Resources/English.lproj/main.nib + + isViewVisible + + + + connectors + + state + + AMLogTabViewSelectedIndex + 0 + libraryState + + actionsMajorSplitViewState + + expandedPosition + 0.0 + subviewState + + 0.000000, 0.000000, 381.000000, 515.000000, NO + 0.000000, 516.000000, 381.000000, 239.000000, NO + + + actionsMinorSplitViewState + + expandedPosition + 0.0 + subviewState + + 0.000000, 0.000000, 163.000000, 515.000000, NO + 164.000000, 0.000000, 217.000000, 515.000000, NO + + + variablesMajorSplitViewState + + expandedPosition + 0.0 + subviewState + + 0.000000, 0.000000, 350.000000, 555.000000, NO + 0.000000, 556.000000, 350.000000, 148.000000, NO + + + variablesMinorSplitViewState + + expandedPosition + 0.0 + subviewState + + 0.000000, 0.000000, 163.000000, 555.000000, NO + 164.000000, 0.000000, 186.000000, 555.000000, NO + + + + majorSplitViewState + + expandedPosition + 0.0 + subviewState + + 0.000000, 0.000000, 381.000000, 800.000000, NO + 382.000000, 0.000000, 619.000000, 800.000000, NO + + + minorSplitViewState + + expandedPosition + 0.0 + subviewState + + 0.000000, 0.000000, 619.000000, 609.000000, NO + 0.000000, 619.000000, 619.000000, 162.000000, NO + + + windowFrame + {{383, 104}, {1000, 877}} + workflowViewScrollPosition + {{0, 0}, {619, 609}} + + workflowMetaData + + serviceApplicationBundleID + com.apple.finder + serviceApplicationPath + /System/Library/CoreServices/Finder.app + serviceInputTypeIdentifier + com.apple.Automator.fileSystemObject.folder + serviceOutputTypeIdentifier + com.apple.Automator.nothing + workflowTypeIdentifier + com.apple.Automator.servicesMenu + + + diff --git a/Install SFU Archives Transfer - External - Mac.zip b/Install SFU Archives Transfer - External - Mac.zip new file mode 100644 index 0000000..0db41ff Binary files /dev/null and b/Install SFU Archives Transfer - External - Mac.zip differ diff --git a/Install SFU Archives Transfer - Internal - Mac.zip b/Install SFU Archives Transfer - Internal - Mac.zip new file mode 100644 index 0000000..7103519 Binary files /dev/null and b/Install SFU Archives Transfer - Internal - Mac.zip differ diff --git a/README.md b/README.md index ff63b04..1b2bc64 100644 --- a/README.md +++ b/README.md @@ -1,88 +1,42 @@ -# Create Bag - -A simple utility to allow users to create a Bag from a directory on their file system. - -This utility is not a replacement for a full-featured tool such as the Library of Congress's Bagger (included in their [Transfer Tools](http://sourceforge.net/projects/loc-xferutils/files/loc-bagger/)). Instead, its purpose is to hide the details of creating a Bag from the end user by providing a simple, familiar user interface. Settings for Bag creation are contained in a configuration file (that would be modified by a network administrator, for example). - -## Features - -* Runs on Linux and Windows (after installation of prerequisites). OSX port is brand new and may still have some bugs. -* On Linux and Windows, uses standard Graphical User Interface file/directory browser to allow user to select which directory to create Bag from; on OSX, shows up as an option in the Finder context menu. -* Configuration options: - * The ability to specify which configuration file to use (as a command-line parameter) - * Definition of the title of the main application window and the file chooser window title - * Definition of custom BagIt tags - * Options to enable some auto-generated BagIt tags (currently Source-Directory and Source-User) - * Choice of checksum algorithms (md5, sha1, sha256, and sha512) - * The option to define a list of "shortcuts" (i.e., links to directories) that appear in the file chooser dialog box. - * The option to copy the contents of the selected directory to specific destination directory before creating the Bag. Note that since the Bag is created from the copy, checksums are generated for the copies, not the original files. - -This last option, the 'create_bag_in' configuration option, is important. If this option is not set, the Bag is created in the directory selected by the user; that is, the directory's contents are rearranged into a Bag structure. The value of this option is set to '/tmp' by default; *Windows users will need to set it manually before running the utility.* - - - - -## Dependencies - -* Python 2.7.8, but should work with any recent version of Python -* [Python GTK+3](http://python-gtk-3-tutorial.readthedocs.org/en/latest/index.html) - * On Linux, Python bindings for GTK+3 should already be installed - * On Windows, install the latest version from http://sourceforge.net/projects/pygobjectwin32/files/?source=navbar. When asked which packages to install, choose GTK+. No other packages in this distribution are necessary for Create Bag to run. - * On OSX, this isn't used. -* [bagit](https://github.com/LibraryOfCongress/bagit-python) - * On all platforms, install with `pip install bagit` (you may need to do easy_install pip first on OSX if it's your first time adding Python packages) - -## Usage - -#### Linux / Windows - -Command line usage is documented here. However, it is possible on both Linux and Windows to create operating-system-specific shortcuts, allowing end users to start the program by clicking or double clicking on a desktop icon. - -Invoke the utility from the command line like this: - -`python createbag.py` - -Create Bag allows you to specify which configuration file to use, as a command-line argument. If run as above, with no argument, the script looks for the file 'config.cfg' in the same directory as the script. To indicate another location for the configuration file, run the script with the file's path as an argument, like this: - -`python createbag.py /path/to/the/config.file` - -When the utility starts, a small window with two buttons appears: - -![Create a Bag](https://dl.dropboxusercontent.com/u/1015702/linked_to/createbag/createbag.png) - -Clicking on the "Choose a directory to create Bag from" will open up a standard file/directory browser: - -![Choose a directory](https://dl.dropboxusercontent.com/u/1015702/linked_to/createbag/choosefolder.png) - -Choosing a directory and clicking on "Create Bag" will create the Bag, after which the following dialog box will appear: - -![Bag created](https://dl.dropboxusercontent.com/u/1015702/linked_to/createbag/bagcreated.png) - -Clicking on "OK" will take the user back to the startup window: - -![Create a Bag](https://dl.dropboxusercontent.com/u/1015702/linked_to/createbag/createbag.png) - -#### OSX - -Unzip (or git clone) to ~/.createbag. Then, create a new Service in the OSX Automator that is designated to receive *folders* in the *Finder*. This Service should perform a single action: "Run Shell Script," using /bin/bash, with the "Pass input" drop-down set to "as arguments". Use this syntax for the Service: - - for f in "$@" - do - python ~/.createbag/createbag.py $f ~/.createbag/config.cfg - done - -Save the Service with a name like "Create Bag". You can then create a bag using this script by right-clicking on any folder in your Finder and selecting "Create Bag" from the context menu: - -![Create a Bag in OS X](https://dl.dropboxusercontent.com/u/1015702/linked_to/createbag/osx.png) - -## To do - -* Compile the utility into native Windows, OS X, and Linux binaries, at which point the utility will be invoked like any other Graphical User Interface application on those operating systems. -* Add better error handling (e.g., options for going back to main application window on Bag creation error, etc.). -* Add error logging, with log path as a configuration option. - -Pull requests implementing these features are welcome. - -## License - -The Unlicense (http://unlicense.org/). Refer to the LICENSE file for more information. +# SFU Archives Deposit -- "Create Bag" Tool + +This is a simple utility, forked from http://github.com/mjordan/createbag, which facilitates deposit of files and folders to SFU Archives by creating a Library of Congress-standard "bagit" container on the local machine and uploading it via SFTP with a small amount of metadata, including transfer credentials and checksums. + +You're welcome to use this in your own institution; all that's required is changing the IP addresses of the upload destinations at various locations in createbag.py. If you want to hack on this, you need Python 2.7 and the official Library of Congress [bagit][] module. + +The tool is designed to work, with GUI functionality, on Windows, OSX, and Linux. The Windows version uses Qt, the OSX version uses CocoaDialog, and the Linux version uses Gtk3 (though it is currently unmaintained and missing a few hooks). + +GUI executables are provided for Windows and OSX, built with pyinstaller and Platypus respectively. The Windows executable runs standalone; the OSX version is installed as an automator hook, detailed below. + + +## Usage + +#### Linux / Windows + +When the utility starts, a small window with two buttons appears: + +![][] + +Clicking on the "Choose a folder to transfer" (may instead simply read "Create bag" in older versions) will open up a standard file/directory browser: + +![][1] + +Choosing a directory and clicking on "Create Bag" will prompt a user for a small amount of metadata (username, password, and deposit number), and then automatically begin creating and transferring the bag in the background. There is currently no progress bar to indicate when the transfer will be complete, but the user will be notified of success or failure with a popup window upon completion. + + +#### OSX + +Download and run either of the zipped OSX installers . You may be prompted to install XCode during the install process; this should be pretty much automatic on newer OSX versions, but can be tricky if you're on Lion or older (let me know). You can then create and transfer bag using this script by right-clicking on any folder in your Finder and selecting "Deposit to SFU Archives" from the context menu (this line may simply read "Create bag" in older versions): + +![][3] + + +## License + +The Unlicense (http://unlicense.org/). Refer to the LICENSE file for +more information. + + [bagit]: https://github.com/LibraryOfCongress/bagit-python + []: https://dl.dropboxusercontent.com/u/1015702/linked_to/createbag/createbag.png + [1]: https://dl.dropboxusercontent.com/u/1015702/linked_to/createbag/choosefolder.png + [3]: https://dl.dropboxusercontent.com/u/1015702/linked_to/createbag/osx.png diff --git a/SFU Archives Transfer - External - Windows.exe b/SFU Archives Transfer - External - Windows.exe new file mode 100644 index 0000000..621cd98 Binary files /dev/null and b/SFU Archives Transfer - External - Windows.exe differ diff --git a/SFU Archives Transfer - Internal - Windows.exe b/SFU Archives Transfer - Internal - Windows.exe new file mode 100644 index 0000000..8f29a2e Binary files /dev/null and b/SFU Archives Transfer - Internal - Windows.exe differ diff --git a/Uninstall SFU Transfer.zip b/Uninstall SFU Transfer.zip new file mode 100644 index 0000000..bbf0a76 Binary files /dev/null and b/Uninstall SFU Transfer.zip differ diff --git a/blacklist.py b/blacklist.py new file mode 100644 index 0000000..8e7788e --- /dev/null +++ b/blacklist.py @@ -0,0 +1,28 @@ +from bottle import get, post, request, run +import re + +@get('/blacklist') +def getblacklist(): + with open("blacklist.txt", "r") as blacklist: + blacklistText = blacklist.read() + return blacklistText + +@post('/blacklist') +def addtoblacklist(): + try: + transfer = request.forms.get("transfer") + session = request.forms.get("session") + username = request.forms.get("username") + checksum = request.forms.get("checksum") + + transfer_meta_path = "/home/" + username + "/deposit_here/" + transfer + "-" + session + "/" + transfer + "-" + session + "-meta.txt" + with open(transfer_meta_path,'rb') as transferMeta: + metaTxt = transferMeta.read() + originalChecksum = re.search('(?<=Checksum: )[a-z0-9]+', metaTxt).group(0) + if originalChecksum == checksum and re.match(r'^\w+$', checksum): + with open("blacklist.txt", "a") as blacklist: + blacklist.write((transfer + "\n")) + except: + pass + +run(host='0.0.0.0', port=8008, debug=True) diff --git a/config.cfg b/config.cfg deleted file mode 100644 index c59c153..0000000 --- a/config.cfg +++ /dev/null @@ -1,47 +0,0 @@ -[Output] -# If 'create_bag_in' is set, the contents of the selected directory will -# be recursively copied into the specified path, and the Bag created there. -# The directory specified in this option will be created if it doesn't exist. -# Note that since the Bag is created from the copy, checksums are generated -# for the copies, not the original files. Also note that if 'create_bag_in' -# is not set, the Bag is created *in place* (that is, the contents of the -# selected directory are rearranged into a Bag structure). -; create_bag_in = C:\Users\Mark\Downloads\bags -create_bag_in = /tmp/createbag/bags - -[UILabels] -# The value of these config options are used as the title of the main -# application window and the file chooser window. -main_window_title = Create Bag from a folder -file_chooser_window_title = Create a Bag - Choose a folder to create Bag from - -[Shortcuts] -# Paths to directories listed here appear as shortcut icons within the file -# chooser dialog box. Provide a comma-separated list of any directory paths -# you want to appear in the file chooser. -; shortcuts = /tmp/, /home/mark/Documents/hacking/bagit -; shortcuts = C:\Users\Mark\Downloads - -[Checksums] -# The bagit Python library produces md5 checksums if none is -# specified. If you want to use another algorithm, or more than -# one, define them here in a comma-separated list. Valid algorithms -# are md5, sha1, sha256, and sha512. -; algorithms = md5,sha1 - -[CustomTags] -# All tag: value pairs you define here will be added -# to bag-info.txt. The bagit Python library always adds -# Bag-Software-Agent, Bagging-Date, and Payload-Oxum. -# Note that auto-generated tags are enabled in the "Other" -# section below. -Contact-Name: Alfred E. Newman -External-Description: Some description would go here. - -[AutogeneratedTags] -# Set this to True to add a 'Source-Directory' tag to bag-info.txt. -add_source_directory_tag = True -# Set this to True to add a 'Source-User' tag to bag-info.txt. The user -# ID is the user that the application is running under. -add_source_user_id_tag = True - diff --git a/createbag.py b/createbag.py index 0bbe091..387ac87 100644 --- a/createbag.py +++ b/createbag.py @@ -2,198 +2,490 @@ GUI tool to create a Bag from a filesystem folder. """ -import ConfigParser import sys import os import shutil import getpass import bagit import platform -if platform.system() != 'Darwin': - # We don't use Gtk on OSX. - from gi.repository import Gtk -else: - # Sets up Cocoadialog for error message popup on OSX. - class cocoaPopup: - - # Change CD_BASE to reflect the location of Cocoadialog on your system. - CD_BASE = "~/.createbag/" - CD_PATH = os.path.join(CD_BASE, "CocoaDialog.app/Contents/MacOS/CocoaDialog") - - def __init__(self, title, message, button): - template = "%s msgbox --title '%s' --text '%s' --button1 '%s'" - self.pipe = os.popen(template % (cocoaPopup.CD_PATH, title, message, button), "w") - - def cocoaError(): - if __name__ == "__main__": - popup = cocoaPopup("Error","Sorry, you can't create a bag here -- you may want to change the config file so that bags are always created in a different output directory, rather than in situ.","OK") - if popup == "1": - popup.close() - sys.exit() - - def cocoaSuccess(bag_dir): - if __name__ == "__main__": - popup = cocoaPopup("Success!","Bag created at %s" % bag_dir,"OK") - if popup == "1": - popup.close() - -# Under Linux and Windows, config file location is first command-line parameter. -# Under OSX, the input directory is the first, the config file is the second. -if platform.system() != 'Darwin': - if len(sys.argv) > 1: - config_file = sys.argv[1] - else: - config_file = './config.cfg' -else: - if len(sys.argv) > 2: - config_file = sys.argv[2] - else: - config_file = './config.cfg' - -config = ConfigParser.ConfigParser() -config.optionxform = str -config.read(config_file) - -# Get custom tags from config file. -bagit_tags = {} -tags = config.options('CustomTags') -for tag in tags: - bagit_tags[tag] = config.get('CustomTags', tag) - -# Get shortcuts from config file. -if config.has_option('Shortcuts', 'shortcuts'): - filechooser_shortcuts = [shortcut.strip() for shortcut in config.get('Shortcuts', 'shortcuts').split(',')] +import random +import string +import re +from time import strftime +import subprocess +from paramiko import SSHClient +from paramiko import AutoAddPolicy +from scp import SCPClient +from distutils.dir_util import copy_tree +import zipfile +import hashlib +import tempfile +from urllib import urlencode +import urllib2 +from zipfile import ZipFile + + +# change this to 1 if building for internal deposits, which use SFU credentials and go to a different server +internalDepositor = 1 +radar = 0 +nobag = 0 + +bagit_checksum_algorithms = ['md5'] + + +session_message = "Please enter Session Number" + +transfer_message = "Please enter Transfer Number" + +if internalDepositor == 0: + username_message = "Username" + password_message = "Password" else: - filechooser_shortcuts = [] + username_message = "SFU Computing ID" + password_message = "SFU Computing password" + +close_session_message = "Is this the final session for this transfer?" + +if radar == 0: + sfu_success_message = "Files have been successfuly transferred to SFU Archives. \nAn archivist will be in contact with you if further attention is needed." -# Get checksum algorithms from config file. -bagit_checksum_algorithms = [] -if config.has_option('Checksums', 'algorithms'): - checksums_string = config.get('Checksums', 'algorithms', 'md5') - bagit_checksum_algorithms = [algo.strip() for algo in checksums_string.split(',')] else: - bagit_checksum_algorithms = ['md5'] - -def directory_check(chosen_folder): - """Prevent the utility from creating a Bag in its own directory.""" - if config.has_option('Output', 'create_bag_in'): - relativized_picker_path = os.path.relpath(chosen_folder, '/') - bag_dir = os.path.join(config.get('Output', 'create_bag_in'), relativized_picker_path) - if os.path.dirname(os.path.realpath(__file__)) == bag_dir: - if platform.system() != 'Darwin': - FolderChooserWindow.GtkError(win) - else: - cocoaError() - else: - if os.path.dirname(os.path.realpath(__file__)) == chosen_folder: - if platform.system() != 'Darwin': - FolderChooserWindow.GtkError(win) - else: - cocoaError() + sfu_success_message = "Files have been successfuly transferred to SFU Library. \nA librarian will be in contact with you if further attention is needed." + password_message = "Please input your SFU Computing password. \nTransfer will commence after clicking OK and you will be notified when it is complete." + +sfu_failure_message = "Transfer did not complete successfully. \nPlease contact Alex Garnett at garnett@sfu.ca for help." + +if platform.system() != 'Darwin' and platform.system() != 'Windows': + # The Linux/Gtk config is currently unmaintained (broken) + from gi.repository import Gtk +elif platform.system() == 'Windows': + from PyQt4 import QtGui, QtCore +elif platform.system() == 'Darwin': + # Sets up Cocoadialog for error message popup on OSX. + CD_PATH = os.path.join("~/.createbag/", "CocoaDialog.app/Contents/MacOS/CocoaDialog") + + def cocoaPopup(boxtype, title, texttype, message, button, buttontext): + template = CD_PATH + " %s --title '%s' '%s' '%s' '%s' '%s'" + cocoa_process = subprocess.Popen(template % (boxtype, title, texttype, message, button, buttontext), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=False) + cocoa_output = cocoa_process.communicate() + cocoa_result = cocoa_output[0].splitlines() + return cocoa_result + + def cocoaError(): + if __name__ == "__main__": + popup = cocoaPopup("msgbox", "Error", "--text", "Sorry, you can't create a bag here -- you may want to change the config file so that bags are always created in a different output directory, rather than in situ.", "--button1", "OK") + if popup == "1": + sys.exit() + + def cocoaSuccess(bag_dir): + if __name__ == "__main__": + popup = cocoaPopup("msgbox", "Success!", "--text", "Bag created at %s" % bag_dir, "--button1", "OK") + + def cocoaTransferSuccess(): + if __name__ == "__main__": + popup = cocoaPopup("msgbox", "SFU Transfer", "--informative-text", sfu_success_message, "--button1", "OK") + + def cocoaTransferError(): + if __name__ == "__main__": + popup = cocoaPopup("msgbox", "SFU Transfer", "--informative-text", sfu_failure_message, "--button1", "OK") + if popup == "1": + sys.exit() + + def cocoaSessionNo(): + if __name__ == "__main__": + popup = cocoaPopup("standard-inputbox", "Session Number", "--informative-text", session_message, "", "") + if popup[0] == "2": + sys.exit() + return popup[1] + + def cocoaTransferNo(): + if __name__ == "__main__": + popup = cocoaPopup("standard-inputbox", "Transfer Number", "--informative-text", transfer_message, "", "") + if popup[0] == "2": + sys.exit() + return popup[1] + + def cocoaUsername(): + if __name__ == "__main__": + popup = cocoaPopup("standard-inputbox", "Username", "--informative-text", username_message, "", "") + if popup[0] == "2": + sys.exit() + return popup[1] + + def cocoaPassword(): + if __name__ == "__main__": + popup = cocoaPopup("secure-standard-inputbox", "Password", "--informative-text", password_message, "", "") + if popup[0] == "2": + sys.exit() + return popup[1] + + def cocoaCloseSession(): + if __name__ == "__main__": + popup = cocoaPopup("yesno-msgbox", "SFU Archives Transfer", "--informative-text", close_session_message, "", "") + if popup[0] == "3": + sys.exit() + # "no" will equal 2 rather than 0 in cocoa, but "yes" still = 1 + return popup[0] + def make_bag(chosen_folder): - """Create a Bag from the files in chosen_folder, and return the directory - the Bag was created in. - """ - if config.getboolean('AutogeneratedTags', 'add_source_directory_tag'): - bagit_tags['Source-Directory'] = chosen_folder - - if config.getboolean('AutogeneratedTags', 'add_source_user_id_tag'): - bagit_tags['Source-User'] = getpass.getuser() - - # If the 'create_bag_in' config option is set, create the Bag from a - # copy of the selected folder. - if config.has_option('Output', 'create_bag_in'): - relativized_picker_path = os.path.relpath(chosen_folder, '/') - bag_dir = os.path.join(config.get('Output', 'create_bag_in'), relativized_picker_path) - try: - shutil.rmtree(bag_dir, True) - shutil.copytree(chosen_folder, bag_dir) - except (IOError, os.error) as shutilerror: - if platform.system() != 'Darwin': - FolderChooserWindow.GtkError(win) - else: - cocoaError() - - # If 'create_bag_in' is not set, create the Bag in the selected directory. - else: - bag_dir = chosen_folder - - # Create the Bag. - try: - bag = bagit.make_bag(bag_dir, bagit_tags, 1, bagit_checksum_algorithms) - except (bagit.BagError, Exception) as e: - if platform.system() != 'Darwin': - FolderChooserWindow.GtkError(win) - else: - cocoaError() - return bag_dir - -# Code within this if block only applies to Linux and Windows, not OSX. -if platform.system() != 'Darwin': - class FolderChooserWindow(Gtk.Window): - - def __init__(self): - Gtk.Window.__init__(self, title = config.get('UILabels', 'main_window_title', 'Create a Bag')) - self.set_border_width(10) - self.move(200, 200) - - box = Gtk.Box(spacing=6) - self.add(box) - self.spinner = Gtk.Spinner() - - choose_folder_button = Gtk.Button("Choose a folder to create Bag from") - choose_folder_button.connect("clicked", self.on_folder_clicked) - box.add(choose_folder_button) - - quit_button = Gtk.Button("Quit") - quit_button.connect("clicked", Gtk.main_quit) - box.add(quit_button) - - def GtkError(self): - not_allowed_message = "\n\nYou are not allowed to run the program on that directory." - error_dialog = Gtk.MessageDialog(self, 0, Gtk.MessageType.ERROR, - Gtk.ButtonsType.OK, "Sorry...") - error_dialog.format_secondary_text(not_allowed_message) - error_dialog.run() - error_dialog.destroy() - raise SystemExit - - def on_folder_clicked(self, widget): - folder_picker_dialog = Gtk.FileChooserDialog( - config.get('UILabels', 'file_chooser_window_title', 'Create a Bag - Choose a folder to create Bag from'), - self, Gtk.FileChooserAction.SELECT_FOLDER) - folder_picker_dialog.set_default_size(800, 400) - folder_picker_dialog.set_create_folders(False) - folder_picker_dialog.add_button("Create Bag", -5) - folder_picker_dialog.add_button("Cancel", -6) - for filechooser_shortcut in filechooser_shortcuts: - folder_picker_dialog.add_shortcut_folder(filechooser_shortcut) - - response = folder_picker_dialog.run() - - if response == -5: - directory_check(folder_picker_dialog.get_filename()) - bag_dir = make_bag(folder_picker_dialog.get_filename()) - folder_picker_dialog.destroy() - - if (bag_dir): - confirmation_dialog = Gtk.MessageDialog(self, 0, Gtk.MessageType.INFO, - Gtk.ButtonsType.OK, "Bag created") - confirmation_dialog.format_secondary_text( - "The Bag for folder %s has been created." % bag_dir) - confirmation_dialog.run() - confirmation_dialog.destroy() - if response == -6: - folder_picker_dialog.destroy() - - win = FolderChooserWindow() - win.connect("delete-event", Gtk.main_quit) - win.show_all() - Gtk.main() - + if nobag == 0: + bag_dir_parent = tempfile.mkdtemp() + if os.path.isdir(bag_dir_parent): + shutil.rmtree(bag_dir_parent) + bag_dir = os.path.join(bag_dir_parent, 'bag') + os.makedirs(bag_dir) + copy_tree(chosen_folder, bag_dir) + + + # Create the Bag. + try: + bag = bagit.make_bag(bag_dir, None, 1, bagit_checksum_algorithms) + except (bagit.BagError, Exception) as e: + if platform.system() == 'Darwin': + cocoaError() + elif platform.system() == 'Windows': + QtChooserWindow.qt_error(ex) + return + else: + FolderChooserWindow.GtkError(win) + + return bag_dir_parent + + else: + return chosen_folder + + +def transfer_manifest(bag_dir, sessionno, transferno, archivesUsername, checksum, metafilename, passwordString): + current_time = strftime("%Y-%m-%d %H:%M:%S") + # plain language manifest + # transfer_metadata = "Your transfer, number " + sessionno + "-" + transferno + ", from user \'" + archivesUsername + "\', from the desktop folder " + clean_bag_dir + ", with md5 checksum " + checksum + ", was successfully received by SFU Archives at " + current_time + "." + # key/value manifest + transfer_metadata = "Transfer Number: " + sessionno + "-" + transferno + "\nUser: " + archivesUsername + "\nChecksum: " + checksum + "\nTime Received: " + current_time + "\nPassword: " + passwordString + + with open(metafilename, 'w') as transfer_metafile: + transfer_metafile.write(transfer_metadata) + + +def generate_password(): + length = 13 + chars = string.ascii_letters + string.digits + '!@#$%^&*()' + random.seed = (os.urandom(1024)) + + passwordString = ''.join(random.choice(chars) for i in range(length)) + return passwordString + + +# need to add error handling to the ssh connection +def check_zip_and_send(bag_dir_parent, sessionno, transferno, archivesUsername, archivesPassword, close_session): + + if nobag == 0: + bag_dir = os.path.join(str(bag_dir_parent), 'bag') + numbered_bag_dir = os.path.join(str(bag_dir_parent), (transferno + "-" + sessionno)) + metafilename = numbered_bag_dir + "-meta.txt" + zipname = shutil.make_archive(numbered_bag_dir, 'zip', bag_dir) + + with open(zipname,'rb') as transferZip: + data = transferZip.read() + checksum = hashlib.md5(data).hexdigest() + + with open(os.path.join(bag_dir, 'manifest-md5.txt'), 'r') as manifestmd5: + bagit_manifest_txt = manifestmd5.read() + filelist = re.sub("\r?\n\S*?\s+", "\n", bagit_manifest_txt) + + with ZipFile(zipname, 'w') as transferZip: + passwordString = generate_password() + transferZip.setpassword(passwordString) + + shutil.rmtree(bag_dir) + + listfilename = numbered_bag_dir + "-list.txt" + with open(listfilename,'w') as transfer_listfile: + transfer_listfile.write(filelist) + + # check transfer number blacklist and post back if OK + get_req = urllib2.Request("http://arbutus.archives.sfu.ca:8008/blacklist") + get_response = urllib2.urlopen(get_req) + blacklist = get_response.read() + blacklist_entries = blacklist.split() + if transferno in blacklist_entries: + if platform.system() == 'Darwin': + cocoaTransferError() + elif platform.system() == 'Windows': + QtChooserWindow.qt_transfer_failure(ex) + return + + values = {'transfer' : transferno, 'session' : sessionno, 'username' : archivesUsername, 'checksum' : checksum} + postdata = urlencode(values) + post_req = urllib2.Request("http://arbutus.archives.sfu.ca:8008/blacklist", postdata) + + + try: + ssh = SSHClient() + ssh.set_missing_host_key_policy(AutoAddPolicy()) + if internalDepositor == 0: + ssh.connect("142.58.136.69", username=archivesUsername, password=archivesPassword, look_for_keys=False) + scp = SCPClient(ssh.get_transport()) + #remote_zip_path = '~/deposit_here/' + sessionno + "-" + transferno + '.zip' + #scp.put(zipname, remote_zip_path) + transfer_manifest(bag_dir, sessionno, transferno, archivesUsername, checksum, metafilename, passwordString) + #remote_meta_path = '~/deposit_here/' + sessionno + "-" + transferno + '-meta.txt' + #scp.put(metafilename, remote_meta_path) + remote_path = '~/deposit_here/' + transferno + "-" + sessionno + scp.put(bag_dir_parent, remote_path, recursive=True) + if close_session == 1: + urllib2.urlopen(post_req) + + elif radar == 1: + ssh.connect("researchdata.sfu.ca", username=archivesUsername, password=archivesPassword, look_for_keys=False) + scp = SCPClient(ssh.get_transport()) + remote_zip_path = '~/.pydiodata/' + os.path.basename(os.path.normpath(bag_dir)) + try: + scp.put(os.path.normpath(bag_dir), remote_zip_path, recursive=True) + except: + ssh.exec_command('mkdir .pydiodata') + scp.put(os.path.normpath(bag_dir), remote_zip_path, recursive=True) + + else: + ssh.connect("pine.archives.sfu.ca", username=archivesUsername, password=archivesPassword, look_for_keys=False) + scp = SCPClient(ssh.get_transport()) + #remote_zip_path = '~/' + sessionno + "-" + transferno + '.zip' + #scp.put(zipname, remote_zip_path) + transfer_manifest(bag_dir, sessionno, transferno, archivesUsername, checksum, metafilename, passwordString) + #remote_meta_path = '~/' + sessionno + "-" + transferno + '-meta.txt' + #scp.put(metafilename, remote_meta_path) + remote_path = '~/' + transferno + "-" + sessionno + scp.put(bag_dir_parent, remote_path, recursive=True) + if close_session == 1: + urllib2.urlopen(post_req) + + + except: + if platform.system() == 'Darwin': + cocoaTransferError() + elif platform.system() == 'Windows': + QtChooserWindow.qt_transfer_failure(ex) + return + + if nobag == 0: + os.remove(zipname) + os.remove(metafilename) + return remote_path + + +# Linux/Gtk-specific code (will work on Windows but not easily) +if platform.system() != 'Darwin' and platform.system() != 'Windows': + class FolderChooserWindow(Gtk.Window): + + def __init__(self): + Gtk.Window.__init__(self, title = config.get('UILabels', 'main_window_title', 'SFU Transfer')) + self.set_border_width(10) + self.move(200, 200) + + box = Gtk.Box(spacing=6) + self.add(box) + self.spinner = Gtk.Spinner() + + choose_folder_button = Gtk.Button("Choose a folder to transfer") + choose_folder_button.connect("clicked", self.on_folder_clicked) + box.add(choose_folder_button) + + quit_button = Gtk.Button("Quit") + quit_button.connect("clicked", Gtk.main_quit) + box.add(quit_button) + + def GtkError(self): + not_allowed_message = "\n\nYou are not allowed to run the program on that directory." + error_dialog = Gtk.MessageDialog(self, 0, Gtk.MessageType.ERROR, + Gtk.ButtonsType.OK, "Sorry...") + error_dialog.format_secondary_text(not_allowed_message) + error_dialog.run() + error_dialog.destroy() + raise SystemExit + + def on_folder_clicked(self, widget): + folder_picker_dialog = Gtk.FileChooserDialog( + config.get('UILabels', 'file_chooser_window_title', 'SFU Transfer - Choose a folder to transfer'), + self, Gtk.FileChooserAction.SELECT_FOLDER) + folder_picker_dialog.set_default_size(800, 400) + folder_picker_dialog.set_create_folders(False) + folder_picker_dialog.add_btuton("Create Bag", -5) + folder_picker_dialog.add_button("Cancel", -6) + for filechooser_shortcut in filechooser_shortcuts: + folder_picker_dialog.add_shortcut_folder(filechooser_shortcut) + + response = folder_picker_dialog.run() + + if response == -5: + bag_dir = make_bag(folder_picker_dialog.get_filename()) + folder_picker_dialog.destroy() + + if (bag_dir): + confirmation_dialog = Gtk.MessageDialog(self, 0, Gtk.MessageType.INFO, + Gtk.ButtonsType.OK, "Bag created") + confirmation_dialog.format_secondary_text( + "The Bag for folder %s has been created." % bag_dir) + confirmation_dialog.run() + confirmation_dialog.destroy() + if response == -6: + folder_picker_dialog.destroy() + + win = FolderChooserWindow() + win.connect("delete-event", Gtk.main_quit) + win.show_all() + Gtk.main() + + +# Windows/Qt-specific code (can also work on Linux but Gtk is nicer) +elif platform.system() == 'Windows': + + class QtChooserWindow(QtGui.QDialog): + + def __init__(self, parent=None): + super(QtChooserWindow, self).__init__(parent) + if parent is None: + self.initUI() + + def initUI(self): + choose_folder_button = QtGui.QPushButton("Choose a folder to transfer", self) + choose_folder_button.clicked.connect(self.showDialog) + choose_folder_button.resize(choose_folder_button.sizeHint()) + choose_folder_button.move(20, 30) + + quit_button = QtGui.QPushButton("Quit", self) + quit_button.clicked.connect(QtCore.QCoreApplication.instance().quit) + quit_button.resize(quit_button.sizeHint()) + quit_button.move(250, 30) + + self.resize(345, 80) + self.center() + self.setWindowTitle('SFU Transfer') + + self.show() + + def center(self): + qr = self.frameGeometry() + cp = QtGui.QDesktopWidget().availableGeometry().center() + qr.moveCenter(cp) + self.move(qr.topLeft()) + + def showDialog(self): + fname = QtGui.QFileDialog.getExistingDirectory(self, 'SFU Transfer - Choose a folder to transfer', '/home') + + bag_dir = make_bag(str(fname)) + + if (bag_dir): + # uncomment the below line and comment the ones after it if not SFU + # self.qt_confirmation(bag_dir) + archivesUsername = self.qt_username(bag_dir) + archivesPassword = self.qt_password(bag_dir) + if radar == 0: + transferno = self.qt_transfer(bag_dir) + sessionno = self.qt_session(bag_dir) + close_session = self.qt_close_session() + else: + sessionno = 0 + transferno = 0 + close_session = 0 + + payload = check_zip_and_send(bag_dir, str(sessionno), str(transferno), str(archivesUsername), str(archivesPassword), close_session) + + if (payload): + self.qt_transfer_success() + + def qt_username(self, bag_dir): + archivesUsername, ok = QtGui.QInputDialog.getText(self, "Username", username_message) + return archivesUsername + + def qt_password(self, bag_dir): + # TODO: obscure text entry + archivesPassword, ok = QtGui.QInputDialog.getText(self, "Password", password_message, 2) + return archivesPassword + + def qt_session(self, bag_dir): + sessionno, ok = QtGui.QInputDialog.getText(self, "Session Number", session_message) + return sessionno + + def qt_transfer(self, bag_dir): + transferno, ok = QtGui.QInputDialog.getText(self, "Transfer Number", transfer_message) + return transferno + + def qt_close_session(self): + close_session_window = QtGui.QMessageBox.question(self, 'SFU Transfer', close_session_message, QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, QtGui.QMessageBox.No) + if close_session_window == QtGui.QMessageBox.Yes: + close_session = 1 + else: + close_session = 0 + return close_session + + def qt_transfer_success(self): + confirmation_window = QtChooserWindow(self) + + confirmation_string = sfu_success_message + confirmation_message = QtGui.QLabel(confirmation_string, confirmation_window) + confirmation_message.move(20, 30) + + confirmation_window.resize(500, 80) + confirmation_window.center() + confirmation_window.setWindowTitle('Success') + + confirmation_window.show() + + def qt_transfer_failure(self): + confirmation_window = QtChooserWindow(self) + + confirmation_string = sfu_failure_message + confirmation_message = QtGui.QLabel(confirmation_string, confirmation_window) + confirmation_message.move(20, 30) + + confirmation_window.resize(500, 80) + confirmation_window.center() + confirmation_window.setWindowTitle('Error') + + confirmation_window.show() + + def qt_confirmation(self, bag_dir): + confirmation_window = QtChooserWindow(self) + + confirmation_string = "The Bag for folder " + bag_dir + " has been created." + confirmation_message = QtGui.QLabel(confirmation_string, confirmation_window) + confirmation_message.move(20, 30) + + confirmation_window.resize(500, 80) + confirmation_window.center() + confirmation_window.setWindowTitle('Bag created') + + confirmation_window.show() + + def qt_error(self): + error_window = QtChooserWindow(self) + + error_message = QtGui.QLabel("Something went wrong! Please open an issue report at http://github.com/axfelix/createbag/issues", error_window) + error_message.move(20, 30) + + error_window.resize(360, 80) + error_window.center() + error_window.setWindowTitle('Sorry') + + error_window.show() + + + app = QtGui.QApplication(sys.argv) + ex = QtChooserWindow() + sys.exit(app.exec_()) + + # OSX-specific code. -else: - directory_check(sys.argv[1]) - bag_dir = make_bag(sys.argv[1]) - cocoaSuccess(bag_dir) +elif platform.system() == 'Darwin': + + bag_dir = make_bag(sys.argv[1]) + # add progress bar code eventually + # comment everything below except the last line if not SFU + archivesUsername = cocoaUsername() + archivesPassword = cocoaPassword() + transferno = cocoaTransferNo() + sessionno = cocoaSessionNo() + close_session = cocoaCloseSession() + if check_zip_and_send(bag_dir, sessionno, transferno, archivesUsername, archivesPassword, close_session): + cocoaTransferSuccess() + # cocoaSuccess(bag_dir) diff --git a/createbag.spec b/createbag.spec new file mode 100644 index 0000000..9f5f38a --- /dev/null +++ b/createbag.spec @@ -0,0 +1,17 @@ +# -*- mode: python -*- +a = Analysis(['createbag.py'], + pathex=['c:\\Users\\Alex\\Dropbox\\Documents\\SFU\\archives\\createbag'], + hiddenimports=[], + hookspath=None, + runtime_hooks=None) +pyz = PYZ(a.pure) +exe = EXE(pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + name='createbag.exe', + debug=False, + strip=None, + upx=True, + console=False ) diff --git a/deposit_monitor.py b/deposit_monitor.py new file mode 100644 index 0000000..7f8cb86 --- /dev/null +++ b/deposit_monitor.py @@ -0,0 +1,36 @@ +# can monitor directories for input to this script like so: + +# inotifywait -m -r -q -e create --format %w%f /home/*/deposit_here | while read line; do if [[ $line == *"-meta.txt" ]]; then echo "Subject: Deposit Report "$(basename $(line)) > deposits.txt; python /home/archivesuser/deposit_monitor.py $line; rsync -az -e ssh $(dirname $line) archivesadmin@pine.archives.sfu.ca:~/arbutus_backups/.; sendmail "radancy@sfu.ca, kas22@sfu.ca" < deposits.txt; rm deposits.txt; fi; done& + + +import hashlib +import re +import sys +from time import strftime + +if not re.search('meta\.txt', sys.argv[1]): + sys.exit() + +zipname = re.sub('-meta\.txt', '.zip', sys.argv[1]) +filelistname = re.sub('-meta\.txt', '-list.txt', sys.argv[1]) + +with open(zipname,'rb') as transferZip: + data = transferZip.read() + checksum = hashlib.md5(data).hexdigest() + +with open(sys.argv[1],'rb') as transferMeta: + metaTxt = transferMeta.read() + originalChecksum = re.search('(?<=Checksum: )[a-z0-9]+', metaTxt).group(0) + +if checksum == originalChecksum: + listnote = "deposit " + sys.argv[1] + " valid " + strftime("%Y-%m-%d %H:%M:%S") + "\n" +else: + listnote = "deposit " + sys.argv[1] + " invalid " + strftime("%Y-%m-%d %H:%M:%S") + "\n" + +with open(filelistname,'rb') as transferList: + listTxt = transferList.read() + +listnote = listnote + listTxt + "\n" + +with open('deposits.txt', 'a') as depositlist: + depositlist.write(listnote) diff --git a/osxinstall.sh b/osxinstall.sh new file mode 100644 index 0000000..83376e8 --- /dev/null +++ b/osxinstall.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +if [ ! -d ~/Library/Services ]; then + mkdir ~/Library/Services +fi + +cp -R CreateBag.workflow ~/Library/Services/CreateBag.workflow +mkdir ~/.createbag +cp createbag ~/.createbag/createbag +chmod +x ~/.createbag/createbag +cp -R CocoaDialog.app ~/.createbag/CocoaDialog.app + +if [ $? -eq 0 ]; then +success=($(~/.createbag/CocoaDialog.app/Contents/MacOS/CocoaDialog msgbox --title "SFU Archives Transfer" --text "Install complete" --informative-text "Hooray! You can now send your file to SFU Archives by right-clicking on a folder in your Finder and selecting 'Transfer to SFU Archives' from the bottom of the menu." --button1 "OK")) +else +failure=($(~/.createbag/CocoaDialog.app/Contents/MacOS/CocoaDialog msgbox --title "SFU Archives Transfer" --text "Install failed" --informative-text "Install did not complete successfully. Please let us know at https://github.com/axfelix/createbag/issues. Sorry!" --button1 "OK")) +fi diff --git a/osxuninstall.sh b/osxuninstall.sh new file mode 100644 index 0000000..40faa85 --- /dev/null +++ b/osxuninstall.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +rm -rf ~/Library/Services/CreateBag.workflow + +~/.createbag/CocoaDialog.app/Contents/MacOS/CocoaDialog msgbox --title "SFU Archives Transfer" --text "Uninstall complete" --informative-text "SFU Archives Transfer has been uninstalled." --button1 "OK" + +rm -rf ~/.createbag diff --git a/sftpuser.sh b/sftpuser.sh new file mode 100644 index 0000000..3cef7ca --- /dev/null +++ b/sftpuser.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +echo "New user name?" +read newuser +echo "New user password?" +read password +if id -u $newuser >/dev/null 2>&1; then + echo "User already exists. Please go to System -> Users and Groups to reconcile existing users." +else + sudo useradd $newuser + echo $newuser":"$password | sudo chpasswd + sudo usermod -a -G sftponly $newuser + sudo mkdir /home/$newuser + sudo mkdir /home/$newuser/deposit_here + sudo chown $newuser /home/$newuser/deposit_here + sudo chmod 755 /home/$newuser/deposit_here + echo "User created. To manage users (including users created by this script), please go to System -> Users and Groups (click the mouse icon in the top-left corner of the desktop)." +fi +killall inotifywait +while read line; do bash ~/inotify_deposit.sh; done < <(inotifywait -m -r -q -e create --format %w%f /home/*/deposit_here) +read \ No newline at end of file