importer.py

#!/usr/bin/env python

import argparse
import os
import json
from collections import OrderedDict
import git

parser = argparse.ArgumentParser(description='cdnjs importer')
parser.add_argument('--lib-name','-l',
                    type=str,
                    help="The library name wanted to import to cdnjs",
                    required=True)
parser.add_argument('--git-url','-g',
                    type=str,
                    help="The git repository url of the library",
                    required=True)
parser.add_argument('--npm-pkg','-n',
                    type=str,
                    help="The npm package name of the library")
parser.add_argument('--lib-author','-a',
                    type=str,
                    help="The author's github account of the library",
                    required=True)
parser.add_argument('--issue-num','-i',
                    type=str,
                    help="The issue number of cdnjs",
                    required=True)

args = parser.parse_args()

# these paths should be set by personal environment
#cdnjs_lib_path = '/tmp/' + args.lib_name
cdnjs_path = '/home/dannyxx001/git_repositories/cdnjs'
cdnjs_lib_path = cdnjs_path + '/ajax/libs/' + args.lib_name
minify_path = '~/git_repositories/web-minify-helper/minify.sh'

# license list approved by cdnjs
with open("license-list.json","r") as license_file:
    license_list = json.load(license_file, object_pairs_hook=OrderedDict)
    license_file.close()

# order dictionary pretend
def ordered_dict_prepend(the_dict, key, value, dict_setitem=dict.__setitem__):
    root = the_dict._OrderedDict__root
    first = root[1]

    if key in the_dict:
        link = the_dict._OrderedDict__map[key]
        link_prev, link_next, _ = link
        link_prev[1] = link_next
        link_next[0] = link_prev
        link[0] = root
        link[1] = first
        root[1] = first[0] = link
    else:
        root[1] = first[0] = the_dict._OrderedDict__map[key] = [root, first, key]
        dict_setitem(the_dict, key, value)

def exist():
    return True

# if the key is wrong, correct it
def correct_key(data, key, correct_key):
    if key in data.keys() and key != correct_key:
        data[correct_key] = data.pop(key)

def handle_package_json(data, filename):
    # update package.json to fit cdnjs
    ordered_dict_prepend(data, 'filename', filename) # pretend filename in first line
    ordered_dict_prepend(data, 'name', data['name']) # let name be the first line
    key_not_use = ['main', 'bugs', 'gitHead', 'dist', 'engines', 'devDependencies', \
                   'scripts', 'directories', 'readme', 'bin', 'jshintConfig', 'eslintConfig', \
                   'styles', 'install', 'typescript', 'browserify', 'browser', 'jam', \
                   'jest', 'scripts', 'peerDependencies', 'contributors', 'gitHEAD', 'issues', \
                   'files', 'ignore', 'engine', 'maintainers']
    cdnjs_lib_version_path = ""

    for key in data.keys():
        if key[0] == '_' or key in key_not_use or not data[key]:
            del data[key]
        elif 'version' in key:
            os.system("mkdir %s/%s" %(cdnjs_lib_path, data[key]))
            cdnjs_lib_version_path = cdnjs_lib_path + '/' + data[key]
            #print cdnjs_lib_version_path
            correct_key(data, key, 'version')
        elif 'author' in key:
            if len(data[key]) == 1:
                data[key] = data[key][data[key].keys()[0]]
            correct_key(data, key, 'author')
        elif 'license' in key:
            #print type(data[key])
            if type(data[key]) is str or type(data[key]) is unicode: # str or unicode (should be only one)
                correct_key(data, key, 'license')
            elif type(data[key]) is dict:                            # dict (should be only one)
                if data[key]['type'] in license_list:
                    data[key] = data[key]['type']
                correct_key(data, key, 'license')
            else:                                                    # list (may contain many licenses)
                if len(data[key]) == 1:
                    data[key] = data[key]['type']
                    correct_key(data, key, 'license')
                else:
                    license_field = []
                    for field in data[key]:
                        if field['type'] in license_list:
                            license_field.append(field['type'])
                        else:
                            license_field.append(field)
                    data['licenses'] = license_field
                    del data[key]
        elif 'homepage' in key:
            if 'io' not in data[key] and 'github' in data[key]:
                del data[key]
            correct_key(data, key, 'homepage')

    return (data, cdnjs_lib_version_path)

def use_npm():
    os.chdir("/tmp")
    #print os.getcwd()
    os.system("mkdir " + cdnjs_lib_path)
    os.system("npm install " + args.npm_pkg + " >/dev/null 2>&1")
    
    lib_pkg_data = OrderedDict()
    filename = args.lib_name + '.min.js'
    cdnjs_lib_version_path = ""
    basepath = ""
    files = []
    files_path = {}

    # handle package.json
    find_package_json = False
    for root, dirnames, filenames in os.walk('/tmp/node_modules'):
        #print root
        #print dirnames
        #print filenames
        for a_file in filenames:
            file_path = os.path.join(root, a_file)
            #print file_path
            if a_file == 'package.json':
                with open(file_path) as data_file:
                    data = json.load(data_file, object_pairs_hook=OrderedDict)
                    data_file.close()
                    if data['name'] != args.lib_name:
                        continue
                    find_package_json = True
                    
                    lib_pkg_data , cdnjs_lib_version_path = handle_package_json(data, filename)
            elif args.lib_name in a_file:
                files_path[a_file] = file_path
                if 'min' in a_file:
                    filename = a_file
                    lib_pkg_data['filename'] = filename

    has_minified = False
    for f in iter(files_path):
        os.system('cp %s %s' %(files_path[f], cdnjs_lib_version_path))
        if has_minified == False and 'min' in f:
            has_minified = True
        else:
            min_file = f.split('.js')[0] + '.min.js'
            if min_file in files_path.keys():
                has_minified = True

        lib_dir = os.path.basename(os.path.dirname(files_path[f]))
        if lib_dir in ['dist', 'build', 'release']:
            basepath = os.path.basename(lib_dir)
            if '**/*' not in files:
                files.append('**/*')
        elif lib_dir == args.lib_name:
            tmp_files = args.lib_name + '*'
            if tmp_files not in files:
                files.append(tmp_files)
    
    if has_minified == False:
        os.system('%s %s >/dev/null 2>&1' %(minify_path, cdnjs_lib_version_path))

    # not exist package.json
    if not find_package_json:
        print "not exist package.json"
        return None

    # add npm auto update config
    lib_pkg_data['npmName'] = args.lib_name
    lib_pkg_data['npmFileMap'] = [OrderedDict({"basePath":basepath})]
    lib_pkg_data['npmFileMap'][0]['files'] = files

    # add library to cdnjs
    os.chdir(cdnjs_lib_path)
    with open('package.json','w') as new_file:
        json.dump(lib_pkg_data, new_file, indent=2, separators=(',',': '))
        new_file.write('\n')
        new_file.close()
    
    os.system("rm -rf /tmp/node_modules")
    return lib_pkg_data['version']

if __name__ == '__main__':
    #print args.lib_name + " " + args.git_url + " " + args.npm_pkg
    #print cdnjs_lib_path
    
    with open(cdnjs_path + "/.git/info/sparse-checkout","a+") as sc:
        new_sc = "/ajax/libs/" + args.lib_name + "/*\n"
        if new_sc not in sc.read():
            sc.write(new_sc)
        sc.close()
    
    repo = git.Repo(cdnjs_path)
    master = repo.heads['master']
    master.checkout()
    origin = repo.remotes['origin']
    origin.pull()
    new_branch = repo.create_head(args.lib_name)
    new_branch.checkout()
    
    if args.npm_pkg:
        lib_version = use_npm()
        commit_message = "add " + args.lib_name + " v" + lib_version + " with autoupdate config"
        if len(commit_message) > 50:
            commit_message = "add " + args.lib_name + " v" + lib_version + "\n\nwith npm autoupdate config"
        commit_message += "\n\nclose #%s, cc @%s" %(args.issue_num, args.lib_author)
        #print commit_message
        repo.git.add(cdnjs_lib_path)
        os.chdir(cdnjs_path)
        if os.system("npm test >/dev/null 2>&1") != 0:
            print "npm test fail"
        else:
            repo.git.commit(message=commit_message)