From 80586b0b5c342e6d8213ba0a1ff0e9edbbffddf7 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 19 Jan 2024 09:59:13 +0000 Subject: [PATCH 01/20] Gruntfile: Switch to python3.11 --- Gruntfile.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gruntfile.js b/Gruntfile.js index 3c215beff..bbc39cbd2 100755 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -5,7 +5,7 @@ const path = require('path'); partial_install_site = "http://www.onezoom.org"; partial_local_install_site = "http://127.0.0.1:8000"; // if you are running a local installation -preferred_python3 = "python3.7"; // in case you have multiple python3 versions installed +preferred_python3 = "python3.11"; // in case you have multiple python3 versions installed web2py_py = path.join(path.dirname(path.dirname(process.cwd())), 'web2py.py'); /** Generate a function to execute a web2py script, handing over all arguments */ From 71c49527c06a34b25c06f13e9affadc6fca44f04 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 19 Jan 2024 10:33:52 +0000 Subject: [PATCH 02/20] package.json: Support Node 18.x To run webpack on node 18 we need to set --openssl-legacy-provider until we can upgrade webpack. --- package.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index a04609aeb..0455e4784 100644 --- a/package.json +++ b/package.json @@ -13,9 +13,10 @@ }, "scripts": { "test": "npx babel-tape-runner OZprivate/rawJS/OZTreeModule/tests/*.js", - "compile_js": "webpack --mode production", - "compile_js_dev:watch": "webpack --watch --config webpack.config.dev.js", - "compile_js_dev": "webpack --mode development" + "comment": "openssl-legacy-provider is needed for node 18 until we upgrade webpack - https://stackoverflow.com/a/69699772", + "compile_js": "NODE_OPTIONS=--openssl-legacy-provider webpack --mode production", + "compile_js_dev:watch": "NODE_OPTIONS=--openssl-legacy-provider webpack --watch --config webpack.config.dev.js", + "compile_js_dev": "NODE_OPTIONS=--openssl-legacy-provider webpack --mode development" }, "repository": { "type": "git", From 4be2b9b549de772d41cf8dd9eb3d57faa862c31c Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 19 Jan 2024 10:48:22 +0000 Subject: [PATCH 03/20] Gruntfile: web2py_configure for web2py housekeeping Automate housekeeping for web2py in a Grunt task --- Gruntfile.js | 14 ++++++++++++-- README.markdown | 4 +--- _COPY_CONTENTS_TO_WEB2PY_DIR/README.markdown | 9 +-------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Gruntfile.js b/Gruntfile.js index bbc39cbd2..80d26f39c 100755 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -29,6 +29,16 @@ module.exports = function (grunt) { grunt.initConfig({ pkg: grunt.file.readJSON('package.json'), exec: { + web2py_configure: { + cwd: "../../", + command: [ + 'git submodule update --init --recursive', + 'ln -sf applications/OZtree/_COPY_CONTENTS_TO_WEB2PY_DIR/routes.py routes.py', + '( [ -d applications/welcome ] && rm -r -- "applications/welcome" || true )', + '( [ -d applications/examples ] && rm -r -- "applications/examples" || true )', + '( [ -f applications/OZtree/private/appconfig.ini ] || { cp applications/OZtree/private/appconfig.ini.example applications/OZtree/private/appconfig.ini ; echo "****** edit private/appconfig.ini"; exit 1; } )', + ].join(" && "), + }, compile_python: { // compile python to make it faster on the server and hence reduce server load. // should probably be run using the same python version as used to run web2py @@ -220,6 +230,6 @@ module.exports = function (grunt) { grunt.registerTask("compile-js_dev", ["exec:compile_js_dev"]); grunt.registerTask("partial-install", ["compile-js", "css", "copy:dev", "curl-dir:get_minlife", "exec:convert_links_to_local"]); grunt.registerTask("partial-local-install", ["compile-js", "css", "copy:dev", "curl-dir:get_local_minlife", "exec:convert_links_to_local"]); - grunt.registerTask("prod", ["clean", "compile-python", "compile-js", "css", "compress", "copy:prod", "make_release_info", "docs"]); - grunt.registerTask("dev", ["clean", "compile-js_dev", "css", "compress", "copy:dev", "make_release_info", "docs"]); + grunt.registerTask("prod", ["clean", "web2py", "compile-python", "compile-js", "css", "compress", "copy:prod", "make_release_info", "docs"]); + grunt.registerTask("dev", ["clean", "web2py", "compile-js_dev", "css", "compress", "copy:dev", "make_release_info", "docs"]); }; diff --git a/README.markdown b/README.markdown index 88f2943e8..c19028883 100755 --- a/README.markdown +++ b/README.markdown @@ -65,7 +65,7 @@ Before anything else, get the OZtree app from [github](https://github.com/OneZoo 2. Install command-line software by running `npm install -g grunt-cli` (you may need to do all this with administrator privileges). 3. Run `npm install` from within the OZtree folder you moved in step 1. then run `grunt dev` (or `grunt prod` if in production mode) - see *"[Building the OneZoom tree viewer](#building-the-onezoom-tree-viewer)"*. 3. [Install](http://dev.mysql.com/downloads/mysql/) & start MySQL, then create a new database (see *"[Setting up the database backend](#setting-up-the-database-backend)"*) -4. Create a appconfig.ini file in `OZtree/private`, with `migrate=1` and which references this database with the appropriate username and password. We also recommend copying the `routes.py` file from `OZtree/_COPY_CONTENTS_TO_WEB2PY_DIR` to the top level of your web2py installation - see *"[Web2py installation](#web2py-installation)"* +4. Edit `private/appconfig.ini` file in `OZtree/private`, with `migrate=1` and with the appropriate database username and password. 5. Fire up a temporary web2py server and visit the main page to create the (empty) database tables - see *"[Starting and shutting down web2py](#starting-and-shutting-down-web2py)"* 6. Load up data into the tables: first create a user and assign it a 'manager' role in the `auth_` tables using the web2py database admin pages, then load the other tables using data from the original OneZoom site (e.g. sent to you via file transfer) - see *"[Filling the database](#filling-the-database)"*. 7. Optimise your installation: @@ -219,8 +219,6 @@ When web2py is run, it will print instructions telling how to shut down the web2 **If this is a new installation** you should now visit `http://127.0.0.1:8000/OZtree/default/` or `https://127.0.0.1:8000/OZtree/default/` to force web2py to create database tables. To load data into the tables, see "Loading Data", below. -Also, if you want to make OneZoom the default application, make a copy of the routes.py file in the folder labelled `_COPY_CONTENTS_TO_WEB2PY_DIR` and place it in the top level web2py directory (see `_COPY_CONTENTS_TO_WEB2PY_DIR/README.markdown`). - Once tables are created, and everything is working, you can set `is_testing = False` in `models/db.py` and `migrate=0` in `private/appconfig.ini`. This will mean that web2py will not make any changes to table structures in the DB, and also that changes to appconfig.ini will require a web2py restart. ### Web2py folder structure diff --git a/_COPY_CONTENTS_TO_WEB2PY_DIR/README.markdown b/_COPY_CONTENTS_TO_WEB2PY_DIR/README.markdown index 7bf3f2c8c..dc513b8d4 100755 --- a/_COPY_CONTENTS_TO_WEB2PY_DIR/README.markdown +++ b/_COPY_CONTENTS_TO_WEB2PY_DIR/README.markdown @@ -1,8 +1 @@ -### Making OneZoom the default application - -To make OneZoom the default web2py app, a copy of the accompanying routes.py file can placed at the root of the web2py folder (at the same level as the `applications` folder). - -### Disabling other applications -You may also wish to delete the `welcome` and `examples` apps (just delete the folders), or disable them from the web2py admin web page (e.g. at http://127.0.0.1:8000/admin). - -You probably shouldn't disable the admin app, as this is the main way of adding users etc to the OneZoom app. It should only be accessible over https (and maybe only locally) anyway, so is not that much of a security risk. \ No newline at end of file +# Python files here will be symlinked by Grunt in web2py_configure From 79ddc36b65bea2ba160af1ba25bcadd20ef3de2e Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 19 Jan 2024 10:57:02 +0000 Subject: [PATCH 04/20] Gruntfile: venv for web2py #676 We need an up-do-date pymysql, get it by wrapping a venv around web2py Add a web2py-run helper to use when running standalone scripts, that will get the web2py setup right. --- Gruntfile.js | 24 ++++++++++++------- .../Utilities/OneOff/make_usernames.py | 2 +- requirements.txt | 1 + tests/unit/test_modules_embed.py | 6 +++-- tests/unit/test_modules_sponsorship.py | 4 ++-- tests/unit/test_modules_usernames.py | 6 +++-- tests/util.py | 2 +- web2py-run | 8 +++++++ 8 files changed, 37 insertions(+), 16 deletions(-) create mode 100644 requirements.txt create mode 100755 web2py-run diff --git a/Gruntfile.js b/Gruntfile.js index 80d26f39c..0e2925c48 100755 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -7,12 +7,13 @@ partial_install_site = "http://www.onezoom.org"; partial_local_install_site = "http://127.0.0.1:8000"; // if you are running a local installation preferred_python3 = "python3.11"; // in case you have multiple python3 versions installed web2py_py = path.join(path.dirname(path.dirname(process.cwd())), 'web2py.py'); +venv_python = path.join(path.dirname(path.dirname(process.cwd())), 'bin/python'); /** Generate a function to execute a web2py script, handing over all arguments */ function exec_web2py_script(script_name, init_args) { return function () { return [ - preferred_python3, + venv_python, web2py_py, '-S OZtree/default', '-M', @@ -37,6 +38,15 @@ module.exports = function (grunt) { '( [ -d applications/welcome ] && rm -r -- "applications/welcome" || true )', '( [ -d applications/examples ] && rm -r -- "applications/examples" || true )', '( [ -f applications/OZtree/private/appconfig.ini ] || { cp applications/OZtree/private/appconfig.ini.example applications/OZtree/private/appconfig.ini ; echo "****** edit private/appconfig.ini"; exit 1; } )', + preferred_python3 + ' -m venv .', + './bin/pip install -r applications/OZtree/requirements.txt', + ].join(" && "), + }, + web2py_start_dev: { + cwd: "../../", + command: [ + '( [ -f oz.crt ] || openssl req -newkey rsa:2048 -x509 -days 365 -nodes -keyout oz.key -subj "/CN=dev.onezoom/" -out oz.crt; )', + venv_python + ' web2py.py -c oz.crt -k oz.key -p 8000 -a pass', ].join(" && "), }, compile_python: { @@ -44,11 +54,7 @@ module.exports = function (grunt) { // should probably be run using the same python version as used to run web2py cwd: "../../", command: - preferred_python3 + ' -c "import gluon.compileapp; gluon.compileapp.compile_application(\'' - + process.cwd() - + '\', skip_failed_views=True)"' - + ' || ' + // If python 3.7 isn't available, use the system-defined python3 instead - 'python3 -c "import gluon.compileapp; gluon.compileapp.compile_application(\'' + venv_python + ' -c "import gluon.compileapp; gluon.compileapp.compile_application(\'' + process.cwd() + '\', skip_failed_views=True)"' }, @@ -69,7 +75,7 @@ module.exports = function (grunt) { } }, make_release_info: { - command: 'git describe --tags > RELEASE_INFO && python3 OZprivate/ServerScripts/Utilities/get_release_name.py RELEASE_INFO >> RELEASE_INFO' + command: 'git describe --tags > RELEASE_INFO && ' + venv_python + ' OZprivate/ServerScripts/Utilities/get_release_name.py RELEASE_INFO >> RELEASE_INFO' }, test_server: { command: function () { @@ -105,7 +111,7 @@ module.exports = function (grunt) { // Any .html file in /static is fair game. See documentation in // https://github.com/OneZoom/OZtree#onezoom-setup command: - "python3 OZprivate/ServerScripts/Utilities/partial_install.py" + + venv_python + " OZprivate/ServerScripts/Utilities/partial_install.py" + " --search " + partial_local_install_site + // replace local urls, for partial local install " --replace " + partial_install_site + " static/*.html" @@ -230,6 +236,8 @@ module.exports = function (grunt) { grunt.registerTask("compile-js_dev", ["exec:compile_js_dev"]); grunt.registerTask("partial-install", ["compile-js", "css", "copy:dev", "curl-dir:get_minlife", "exec:convert_links_to_local"]); grunt.registerTask("partial-local-install", ["compile-js", "css", "copy:dev", "curl-dir:get_local_minlife", "exec:convert_links_to_local"]); + grunt.registerTask("web2py", ["exec:web2py_configure"]); grunt.registerTask("prod", ["clean", "web2py", "compile-python", "compile-js", "css", "compress", "copy:prod", "make_release_info", "docs"]); grunt.registerTask("dev", ["clean", "web2py", "compile-js_dev", "css", "compress", "copy:dev", "make_release_info", "docs"]); + grunt.registerTask("start-dev", ['exec:web2py_start_dev']); }; diff --git a/OZprivate/ServerScripts/Utilities/OneOff/make_usernames.py b/OZprivate/ServerScripts/Utilities/OneOff/make_usernames.py index aed707ef4..9c53d956f 100644 --- a/OZprivate/ServerScripts/Utilities/OneOff/make_usernames.py +++ b/OZprivate/ServerScripts/Utilities/OneOff/make_usernames.py @@ -1,6 +1,6 @@ """ Run from the OZtree directory as -python3 ../../web2py.py -S OZtree -M -R applications/OZtree/OZprivate/ServerScripts/Utilities/OneOff/make_usernames.py +./web2py-run OZprivate/ServerScripts/Utilities/OneOff/make_usernames.py After 2 passes, will allocate usernames of remaining unallocated reservations using a species name plus year. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..77947e063 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +pymysql==1.1.0 diff --git a/tests/unit/test_modules_embed.py b/tests/unit/test_modules_embed.py index fc0b96cb9..0189485ae 100644 --- a/tests/unit/test_modules_embed.py +++ b/tests/unit/test_modules_embed.py @@ -1,6 +1,8 @@ """ -Run with -python3 web2py.py -S OZtree -M -R applications/OZtree/tests/unit/test_modules_embed.py +Run with:: + + ./web2py-run tests/unit/test_modules_embed.py + """ import re import unittest diff --git a/tests/unit/test_modules_sponsorship.py b/tests/unit/test_modules_sponsorship.py index 717c68ae3..3438593a1 100644 --- a/tests/unit/test_modules_sponsorship.py +++ b/tests/unit/test_modules_sponsorship.py @@ -1,7 +1,7 @@ """ -Run with +Run with:: -python3 web2py.py -S OZtree -M -R applications/OZtree/tests/unit/test_modules_sponsorship.py + ./web2py-run tests/unit/test_modules_sponsorship.py Note you should make sure prices are set before running tests (manage/SET_PRICES.html) """ diff --git a/tests/unit/test_modules_usernames.py b/tests/unit/test_modules_usernames.py index ccbfc26f9..a72e7a207 100644 --- a/tests/unit/test_modules_usernames.py +++ b/tests/unit/test_modules_usernames.py @@ -1,6 +1,8 @@ """ -Run with -python3 web2py.py -S OZtree -M -R applications/OZtree/tests/unit/test_modules_username.py +Run with:: + + ./web2py-run tests/unit/test_modules_username.py + """ import unittest diff --git a/tests/util.py b/tests/util.py index 3f5b30649..2ea1d8e3e 100644 --- a/tests/util.py +++ b/tests/util.py @@ -69,7 +69,7 @@ def __init__(self, appconfig_file=None): self.pid = None if self.is_local(): print("> starting web2py") - cmd = ['python3', os.path.join(web2py_app_dir, '..','..','web2py.py'), '-Q', '-i', ip, '-p', port, '-a', 'pass'] + cmd = [os.path.join(web2py_app_dir, '..','..','bin', 'python3'), os.path.join(web2py_app_dir, '..','..','web2py.py'), '-Q', '-i', ip, '-p', port, '-a', 'pass'] if appconfig_file is not None: cmd += ['--args', appconfig_file] self.pid = subprocess.Popen(cmd) diff --git a/web2py-run b/web2py-run new file mode 100755 index 000000000..c17db9934 --- /dev/null +++ b/web2py-run @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu + +SCRIPT="applications/OZtree/$1" +shift + +cd ../../ +exec ./bin/python3 web2py.py -S OZtree -M -R "${SCRIPT}" --args $* From a73c25e5a7efa30a90a505545b3e2a6f34073a0f Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 19 Jan 2024 11:21:51 +0000 Subject: [PATCH 05/20] controllers/API: Remove search_log This will impact search performance, by forcing all search queries to go through the unique index before searching. Remove, replace with google analytics later. --- OZprivate/ServerScripts/SQL/create_db_indexes.sql | 4 ---- controllers/API.py | 10 ---------- models/db.py | 6 ------ private/appconfig.ini.example | 1 - 4 files changed, 21 deletions(-) diff --git a/OZprivate/ServerScripts/SQL/create_db_indexes.sql b/OZprivate/ServerScripts/SQL/create_db_indexes.sql index 851b6b815..339716d16 100644 --- a/OZprivate/ServerScripts/SQL/create_db_indexes.sql +++ b/OZprivate/ServerScripts/SQL/create_db_indexes.sql @@ -37,7 +37,6 @@ call MakeFullUnicode('images_by_ott', 'rights'); call MakeFullUnicode('images_by_ott', 'licence'); call MakeFullUnicode('images_by_name', 'rights'); call MakeFullUnicode('images_by_name', 'licence'); -call MakeFullUnicode('search_log', 'search_string'); # note make sure that the name column in vernacular_by_name and the name column in ordered_leaves and ordered_nodes are of the same character set otherwise search can get incredibly slow even with indexes. @@ -213,9 +212,6 @@ CREATE FULLTEXT INDEX ft_user_sponsor_info_index ON reservations (user_more_info DROP INDEX ipni_index ON PoWO; CREATE INDEX ipni_index ON PoWO (ipni_int) USING HASH; -DROP INDEX string_index ON search_log; -CREATE INDEX string_index ON search_log (search_string) USING HASH; - DROP INDEX identifier_index ON partners; CREATE INDEX identifier_index ON partners (partner_identifier) USING HASH; diff --git a/controllers/API.py b/controllers/API.py index bf1e3982a..46fe0ca6b 100755 --- a/controllers/API.py +++ b/controllers/API.py @@ -125,16 +125,6 @@ def search_node(): session.forget(response) response.headers["Access-Control-Allow-Origin"] = '*' searchFor = make_unicode(request.vars.query or '') - try: - if myconf.take('general.log_search_strings'): - if request.vars.no_log: - #'no_log' flag set: this is probably us blatting the search for testing purposes - pass - else: - db.search_log.update_or_insert(db.search_log.search_string==searchFor, search_string=searchFor, search_count=db.search_log.search_count+1) - - except: - pass res1 = search_for_name() if len(res1['leaf_hits']) + len(res1['node_hits']) <15: try: diff --git a/models/db.py b/models/db.py index d85678a70..1c6e287e1 100755 --- a/models/db.py +++ b/models/db.py @@ -660,12 +660,6 @@ Field('leaf_click_count', type='integer'), format = '%(ott)s', migrate=is_testing) -# this table collects a list of search terms so we can optimise search -db.define_table('search_log', - Field('search_string', type='string', notnull=True, unique=True, length=name_length_chars), #this should be utf8mb4 - Field('search_count', type='integer', notnull=True), - format = '%(search_string)s', migrate=is_testing) - # This table buffers recently 'visited' EoL taxa (visited through the window popup or via the copyright link) # taxa in this table are stored until at least 1 minute after the taxon is visited, and then read by the EOL update # script (EoLQueryPicsNames.py) to check for updates to the crop location, ratings, etc. Once checked, the taxon diff --git a/private/appconfig.ini.example b/private/appconfig.ini.example index a873376c1..d92c84e1d 100755 --- a/private/appconfig.ini.example +++ b/private/appconfig.ini.example @@ -29,7 +29,6 @@ separator = url = https://www.sandbox.paypal.com [general] -;log_search_strings = 1 [images] ; * url_base: get thumbnail images from this source. If not From 40f9fe1d6e5a0d33fe48ed55ed11f3f301266ff7 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 19 Jan 2024 15:01:18 +0000 Subject: [PATCH 06/20] models/db: is_testing iff running under rocket Instead of having to tweak is_testing for production use, check the request environment to see what server we're using. --- README.markdown | 4 ++-- models/db.py | 36 +++++++++++++++--------------------- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/README.markdown b/README.markdown index c19028883..6a340cbf7 100755 --- a/README.markdown +++ b/README.markdown @@ -70,7 +70,7 @@ Before anything else, get the OZtree app from [github](https://github.com/OneZoo 6. Load up data into the tables: first create a user and assign it a 'manager' role in the `auth_` tables using the web2py database admin pages, then load the other tables using data from the original OneZoom site (e.g. sent to you via file transfer) - see *"[Filling the database](#filling-the-database)"*. 7. Optimise your installation: * create indexes on the tables by running the SQL script in `OZtree/OZprivate/ServerScripts/SQL/create_db_indexes.sql`. You can do this, for example, by running `SOURCE /path/to/OZtree/OZprivate/ServerScripts/SQL/create_db_indexes.sql` within a mysql client. - * set `is_testing = False` in `models/db.py` and `migrate=0` in appconfig.ini. + * set `migrate=0` in appconfig.ini. ## Downloading the OZtree app @@ -219,7 +219,7 @@ When web2py is run, it will print instructions telling how to shut down the web2 **If this is a new installation** you should now visit `http://127.0.0.1:8000/OZtree/default/` or `https://127.0.0.1:8000/OZtree/default/` to force web2py to create database tables. To load data into the tables, see "Loading Data", below. -Once tables are created, and everything is working, you can set `is_testing = False` in `models/db.py` and `migrate=0` in `private/appconfig.ini`. This will mean that web2py will not make any changes to table structures in the DB, and also that changes to appconfig.ini will require a web2py restart. +Once tables are created, and everything is working, you can set `migrate=0` in `private/appconfig.ini`. This will mean that web2py will not make any changes to table structures in the DB ### Web2py folder structure diff --git a/models/db.py b/models/db.py index 1c6e287e1..93ba3fec0 100755 --- a/models/db.py +++ b/models/db.py @@ -10,12 +10,6 @@ ## if SSL/HTTPS is properly configured and you want all HTTP requests to ## be redirected to HTTPS, uncomment the line below: # request.requires_https() -#set the default language -T.set_current_languages('en', 'en-en') -#ALL pages can set ?lang=XXX to override the browser default for translating strings -if request.vars.lang: - T.force(request.vars.lang) - ## app configuration made easy. Look inside private/appconfig.ini from gluon.contrib.appconfig import AppConfig @@ -25,26 +19,26 @@ ## Useful global variables ######################################################################### -## once in production, set is_testing=False to gain optimizations -## this will also set migration=False for all tables, so that the DB table definitions are fixed -is_testing = True +# Running under rocket --> is_testing is true +is_testing = (request.env.server_software or '').lower().startswith('rocket') -## get config params etc -if is_testing: +## Read configuration +if is_testing and len(request.env.cmd_options.args) > 1 and os.path.isfile(request.env.cmd_options.args[-1]): # For unit testing, we might want to load a different appconfig.ini file, which can # be passed in to the rocket server as the last arg on the command-line. # (on the main server this is not used, and we default back to appconfig.ini - try: - if os.path.isfile(request.env.cmd_options.args[-1]): - myconf = AppConfig(request.env.cmd_options.args[-1], reload=True) - else: - raise IOError("No such file") - except (IOError, IndexError, AttributeError): - myconf = AppConfig(reload=True) #changes to appconfig.ini do not require restart - T.is_writable = True #allow translators to add new languages e.g. on the test (beta) site, but not on prod + myconf = AppConfig(request.env.cmd_options.args[-1], reload=is_testing) else: - myconf = AppConfig() #faster to read once and never re-update - T.is_writable = False + # NB: When running under rocket, re-load config every request with is_testing + myconf = AppConfig(reload=is_testing) + +## Configure i18n +T.set_current_languages('en', 'en-en') +# Allow translators to add new languages e.g. on the test (beta) site, but not on prod +T.is_writable = is_testing +#ALL pages can set ?lang=XXX to override the browser default for translating strings +if request.vars.lang: + T.force(request.vars.lang) try: thumb_base_url = myconf.take('images.url_base') From 64ae1daf5995b15b1a3ca448166bb5d9a8d8c28a Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 19 Jan 2024 15:09:52 +0000 Subject: [PATCH 07/20] models/db: Database session-handling #82 Not that we should be using sessions much, if at all. --- models/db.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/models/db.py b/models/db.py index 93ba3fec0..9a18ddd0c 100755 --- a/models/db.py +++ b/models/db.py @@ -150,6 +150,9 @@ auth.settings.registration_requires_approval = False auth.settings.reset_password_requires_verification = True +## Configure session handling: http://web2py.com/books/default/chapter/29/13/deployment-recipes#Sessions-in-database +session.connect(request, response, db) + ##restrict site to only logged in users ## https://groups.google.com/forum/#!topic/web2py/0j92-sPp4bc ##NB: useful url to add a guest user programmatically http://stackoverflow.com/questions/35504306/web2py-how-to-programmatically-register-users/35518991 From 8a2246d94d673a240e5e2c844e9823a1803efbbc Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 19 Jan 2024 16:52:33 +0000 Subject: [PATCH 08/20] install: Install scripts for nginx/supervisord Scripts to configure nginx/supervisord --- Gruntfile.js | 1 + install-nginx.sh | 260 +++++++++++++++++++++++++++++++++++++++++ install-supervisord.sh | 121 +++++++++++++++++++ requirements.txt | 1 + 4 files changed, 383 insertions(+) create mode 100755 install-nginx.sh create mode 100755 install-supervisord.sh diff --git a/Gruntfile.js b/Gruntfile.js index 0e2925c48..920702373 100755 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -35,6 +35,7 @@ module.exports = function (grunt) { command: [ 'git submodule update --init --recursive', 'ln -sf applications/OZtree/_COPY_CONTENTS_TO_WEB2PY_DIR/routes.py routes.py', + 'ln -sf handlers/wsgihandler.py handler.py', '( [ -d applications/welcome ] && rm -r -- "applications/welcome" || true )', '( [ -d applications/examples ] && rm -r -- "applications/examples" || true )', '( [ -f applications/OZtree/private/appconfig.ini ] || { cp applications/OZtree/private/appconfig.ini.example applications/OZtree/private/appconfig.ini ; echo "****** edit private/appconfig.ini"; exit 1; } )', diff --git a/install-nginx.sh b/install-nginx.sh new file mode 100755 index 000000000..fe73320cd --- /dev/null +++ b/install-nginx.sh @@ -0,0 +1,260 @@ +#!/usr/bin/env bash +set -eux +PROJECT_PATH="${PROJECT_PATH-$(dirname "$(readlink -f "$0")")}" # The full project path +WEB2PY_PATH="$(dirname $(dirname "$PROJECT_PATH"))" +WEB2PY_NAME="${WEB2PY_NAME-$(basename ${WEB2PY_PATH})}" # Directory web2py lives in, will be unique per installation +CERT_DIR="/var/lib/dehydrated/certs" + +WWW_SERVER_NAME="${WEB2PY_NAME}" # Assume we checked out web2py in /.../www.onezoom.org +WWW_IMAGES_SERVER_NAME="$(echo ${WWW_SERVER_NAME} | sed 's/^w*/images/')" # images.onezoom.org or imagesdev.onezoom.org + +[ -d "/etc/nginx" ] && NGINX_PATH="/etc/nginx" +[ -d "/usr/local/etc/nginx" ] && NGINX_PATH="/usr/local/etc/nginx" +mkdir -p "${NGINX_PATH}/conf.d/" +NGINX_LOG_PATH="/var/log/nginx" +[ -d "/var/db/acme/live/" ] && NGINX_CERT_PATH="/var/db/acme/live/" +[ -d "/var/lib/dehydrated/certs" ] && NGINX_CERT_PATH="/var/lib/dehydrated/certs" +NGINX_DHPARAM_PATH="${NGINX_PATH}/dhparam.pem" +[ -d "/var/db/acme/live" ] && NGINX_CHALLENGE_PATH="/var/db/acme/live" +[ -d "/var/lib/dehydrated/acme-challenges" ] && NGINX_CHALLENGE_PATH="/var/lib/dehydrated/acme-challenges" + +# Generate NGINX_DHPARAM +[ -e "${NGINX_DHPARAM_PATH}" ] || openssl dhparam -out "${NGINX_DHPARAM_PATH}" 4096 + +# Self-signed bootstrap-cert +for SN in onezoom.org ${WWW_SERVER_NAME} ${WWW_IMAGES_SERVER_NAME}; do + mkdir -p "${NGINX_CERT_PATH}/${SN}" + if [ ! -e "${NGINX_CERT_PATH}/${SN}/privkey.pem" ]; then + openssl req -x509 -newkey rsa:4096 -sha256 -days 365 -nodes \ + -keyout "${NGINX_CERT_PATH}/${SN}/privkey-ss.pem" \ + -out "${NGINX_CERT_PATH}/${SN}/fullchain-ss.pem" \ + -subj "/CN=${SN}" \ + -addext "subjectAltName = DNS:selfsigned.${SN}" + ln -rs ${NGINX_CERT_PATH}/${SN}/privkey-ss.pem ${NGINX_CERT_PATH}/${SN}/privkey.pem + ln -rs ${NGINX_CERT_PATH}/${SN}/fullchain-ss.pem ${NGINX_CERT_PATH}/${SN}/fullchain.pem + ln -rs ${NGINX_CERT_PATH}/${SN}/fullchain-ss.pem ${NGINX_CERT_PATH}/${SN}/chain.pem + fi +done + +# Create NGINX config +cat < ${NGINX_PATH}/nginx.conf +#### Generated by $0 - DO NOT EDIT + +events {} + +http { + include mime.types; + default_type application/octet-stream; + + # Enable Gzip + gzip on; + gzip_http_version 1.0; + gzip_comp_level 2; + gzip_min_length 1100; + gzip_buffers 4 8k; + gzip_proxied any; + gzip_types + # text/html is always compressed by HttpGzipModule + text/css + text/javascript + text/xml + text/plain + text/x-component + text/json + application/javascript + application/json + application/xml + application/rss+xml + font/truetype + font/opentype + application/vnd.ms-fontobject + image/svg+xml; + gzip_proxied expired no-cache no-store private auth; + gzip_disable "MSIE [1-6]\."; + gzip_vary on; + + keepalive_timeout 65; + + server_names_hash_bucket_size 128; + + include /etc/nginx/conf.d/*.conf; +} +EOF + +cat < ${NGINX_PATH}/conf.d/onezoom.org.conf +server { + listen 80; + listen [::]:80; + listen 443 ssl http2; + listen [::]:443 ssl http2; + + server_name server_name onezoom.org default; + + location /.well-known/acme-challenge { + allow all; + default_type text/plain; + alias ${NGINX_CHALLENGE_PATH}; + } + ssl_certificate ${NGINX_CERT_PATH}/onezoom.org/fullchain.pem; + ssl_certificate_key ${NGINX_CERT_PATH}/onezoom.org/privkey.pem; + ssl_trusted_certificate ${NGINX_CERT_PATH}/onezoom.org/fullchain.pem; + ssl_dhparam ${NGINX_DHPARAM_PATH}; + + return 301 \$scheme://www.onezoom.org\$request_uri; +} +EOF + +cat < ${NGINX_PATH}/${WEB2PY_NAME}_static_include.inc +#### Generated by $0 - DO NOT EDIT + +alias ${PROJECT_PATH}/static/; + +#the directives used for static pages on OneZoom +location ~ /FinalOutputs/data/ { + # add_header 'X-static-gzipping' 'on' always; + gzip_static on; + #files in /data/ (e.g. the topology) have timestamps, so never change, and browsers can always use cache + expires max; + add_header Cache-Control "public"; +} + +location ~* \.(?:jpg|jpeg|gif|png|ico|gz|svg)\$ { + #cache images for a little while, even though we also cache them in the js. + #10 mins allows e.g. new crops to show up. + expires 10m; + access_log off; + add_header Cache-Control "public"; +} + +location ~ \.(js|css|html)\$ { + # add_header 'X-static-gzipping' 'on' always; + gzip_static on; + #cache the static js and html, but only for a bit, in case we implement changes + expires 30m; + add_header Cache-Control "public"; +} + +location ~* /(\w+/)?static/trees/[^/]+/ { + # static trees with a trailing slash need to be trimmed so that e.g. + # static/trees/AT/@Homo_sapiens is not seen as a request for a file called '@Homo_sapiens' + # see http://stackoverflow.com/questions/39519355 + rewrite ^(.*/static/trees/[^/]+)/ \$1 last; + return 404; +} +EOF + +cat < ${NGINX_PATH}/conf.d/${WWW_SERVER_NAME}.conf +#### Generated by $0 - DO NOT EDIT + +upstream uwsgi_${WEB2PY_NAME} { + least_conn; + server unix:///var/run/uwsgi/${WEB2PY_NAME}_uwsgi0.sock; + server unix:///var/run/uwsgi/${WEB2PY_NAME}_uwsgi1.sock; + server unix:///var/run/uwsgi/${WEB2PY_NAME}_uwsgi2.sock; + server unix:///var/run/uwsgi/${WEB2PY_NAME}_uwsgi3.sock; + server unix:///var/run/uwsgi/${WEB2PY_NAME}_uwsgi4.sock; +} + +server { + listen 80; + listen [::]:80; + listen 443 ssl http2; + listen [::]:443 ssl http2; + + server_name ${WWW_SERVER_NAME}; + + if (\$scheme != "https") { + return 301 https://\$server_name\$request_uri; + } + location /.well-known/acme-challenge { + allow all; + default_type text/plain; + alias ${NGINX_CHALLENGE_PATH}; + } + ssl_certificate ${NGINX_CERT_PATH}/${WWW_SERVER_NAME}/fullchain.pem; + ssl_certificate_key ${NGINX_CERT_PATH}/${WWW_SERVER_NAME}/privkey.pem; + ssl_trusted_certificate ${NGINX_CERT_PATH}/${WWW_SERVER_NAME}/fullchain.pem; + ssl_dhparam ${NGINX_DHPARAM_PATH}; + + # Generated by https://ssl-config.mozilla.org/ + ssl_session_timeout 1d; + ssl_session_cache shared:MozSSL:10m; # about 40000 sessions + ssl_session_tickets off; + + # intermediate configuration + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305; + ssl_prefer_server_ciphers off; + + server_tokens off; + access_log ${NGINX_LOG_PATH}/${WWW_SERVER_NAME}.access.log combined buffer=32k flush=5m; + error_log ${NGINX_LOG_PATH}/${WWW_SERVER_NAME}.error.log; + + #cache filehandles on the server side to max 2000 files, hopefully mostly images + open_file_cache max=2000 inactive=20s; + open_file_cache_valid 60s; + open_file_cache_min_uses 3; + open_file_cache_errors off; + + index index.htm; + location /static/ { + include ${WEB2PY_NAME}_static_include.inc; + } + + location /OZtree/static/ { + include ${WEB2PY_NAME}_static_include.inc; + } + + location / { + location ~ \.json { + #don't log API (json) requests + access_log off; + uwsgi_pass uwsgi_${WEB2PY_NAME}; + } + uwsgi_pass uwsgi_${WEB2PY_NAME}; + include uwsgi_params; + } +} +EOF + +cat < ${NGINX_PATH}/conf.d/${WWW_IMAGES_SERVER_NAME}.conf +#### Generated by $0 - DO NOT EDIT + +server { + listen 80; + listen [::]:80; + listen 443 ssl http2; + listen [::]:443 ssl http2; + + server_name ${WWW_IMAGES_SERVER_NAME}; + server_tokens off; + + location /.well-known/acme-challenge { + allow all; + default_type text/plain; + alias ${NGINX_CHALLENGE_PATH}; + } + ssl_certificate ${NGINX_CERT_PATH}/${WWW_IMAGES_SERVER_NAME}/fullchain.pem; + ssl_certificate_key ${NGINX_CERT_PATH}/${WWW_IMAGES_SERVER_NAME}/privkey.pem; + ssl_trusted_certificate ${NGINX_CERT_PATH}/${WWW_IMAGES_SERVER_NAME}/fullchain.pem; + ssl_dhparam ${NGINX_DHPARAM_PATH}; + + access_log ${NGINX_LOG_PATH}/${WWW_IMAGES_SERVER_NAME}.access.log combined buffer=32k flush=5m; + error_log ${NGINX_LOG_PATH}/${WWW_IMAGES_SERVER_NAME}.error.log; + + #cache filehandles on the server side to max 2000 files, hopefully mostly images + open_file_cache max=2000 inactive=20s; + open_file_cache_valid 60s; + open_file_cache_min_uses 3; + open_file_cache_errors off; + + location / { + access_log off; + expires 10m; + add_header Cache-Control "public"; + root ${PROJECT_PATH}/static/FinalOutputs/img/; + } +} +EOF + +nginx -t diff --git a/install-supervisord.sh b/install-supervisord.sh new file mode 100755 index 000000000..25388ad43 --- /dev/null +++ b/install-supervisord.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +set -eu +PROJECT_PATH="${PROJECT_PATH-$(dirname "$(readlink -f "$0")")}" # The full project path +WEB2PY_PATH="$(dirname $(dirname "$PROJECT_PATH"))" +WEB2PY_NAME="${WEB2PY_NAME-$(basename ${WEB2PY_PATH})}" # Directory web2py lives in, will be unique per installation +DEPLOY_USER="$(stat -c '%U' $0)" + +APP_USER="www" +APP_GROUP="www" +WWW_SERVER_NAME="${WEB2PY_NAME}" + +# NB: Yan says we should write to web2py_path & gluon. Seems risky +for DIR in "${PROJECT_PATH}/errors" \ + "${PROJECT_PATH}/databases" \ + "${PROJECT_PATH}/sessions" \ + "${PROJECT_PATH}/uploads" \ + "${WEB2PY_PATH}/logs" \ + "${WEB2PY_PATH}/deposit" \ + "/var/run/uwsgi"; do + mkdir -p -- "${DIR}" + chown -R ${APP_USER} "${DIR}" + chmod g+w "${DIR}" +done + +[ -d "/etc/supervisor" ] && SUPERVISORD_CONF_PATH="/etc/supervisor" +[ -d "/usr/local/etc/supervisord" ] && SUPERVISORD_CONF_PATH="/usr/local/etc/supervisord" +mkdir -p "${SUPERVISORD_CONF_PATH}/supervisord.conf.d/" +mkdir -p "/var/run/supervisor" + +SUPERVISORD_LOG_PATH="/var/log/supervisord" +mkdir -p "${SUPERVISORD_LOG_PATH}" + +cat < "${SUPERVISORD_CONF_PATH}/supervisord.conf" +; **** Generated by $0 - DO NOT EDIT +; +[unix_http_server] +file=/var/run/supervisor/supervisor.sock ; (the path to the socket file) + +[supervisord] +logfile=${SUPERVISORD_LOG_PATH}/supervisord.log ; (main log file) +logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB) +logfile_backups=10 ; (num of main logfile rotation backups;default 10) +loglevel=info ; (log level;default info; others: debug,warn,trace) +pidfile=/var/run/supervisor/supervisord.pid ; (supervisord pidfile;default supervisord.pid) +nodaemon=false ; (start in foreground if true;default false) +minfds=1024 ; (min. avail startup file descriptors;default 1024) +minprocs=200 ; (min. avail process descriptors;default 200) + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///var/run/supervisor/supervisor.sock ; use a unix:// URL for a unix socket + +[include] +files = supervisord.conf.d/*.conf +EOF + +cat < "${SUPERVISORD_CONF_PATH}/supervisord.conf.d/${WEB2PY_NAME}.conf" +; **** Generated by $0 - DO NOT EDIT +; +[group:${WEB2PY_NAME}] +programs=${WEB2PY_NAME}_uwsgi,${WEB2PY_NAME}_background_tasks,${WEB2PY_NAME}_session_cleanup + +[program:${WEB2PY_NAME}_uwsgi] +directory=${WEB2PY_PATH} +command=${WEB2PY_PATH}/bin/uwsgi + -s /var/run/uwsgi/%(program_name)s%(process_num)d.sock + --chmod-socket=666 + --need-app + --disable-logging + --master + --home=${WEB2PY_PATH} + --wsgi-file handler.py + --processes 1 + --threads 10 + --uid "${APP_USER}" + --gid "${APP_GROUP}" +stdout_logfile=${SUPERVISORD_LOG_PATH}/%(program_name)s_stdout.log +stdout_logfile_maxbytes=10MB +stdout_logfile_backups=10 +stderr_logfile=${SUPERVISORD_LOG_PATH}/%(program_name)s_stderr.log +stderr_logfile_maxbytes=10MB +stderr_logfile_backups=10 +startsecs=10 +stopsignal=QUIT +stopasgroup=true +killasgroup=true +process_name=%(program_name)s%(process_num)d +numprocs=5 + +[program:${WEB2PY_NAME}_background_tasks] +directory=${WEB2PY_PATH} +user=${APP_USER} +group=${APP_GROUP} +command=$(which bash) -c "${WEB2PY_PATH}/bin/python web2py.py -S OZtree/default -M -e -R applications/OZtree/private/background_tasks.py --args ${WWW_SERVER_NAME} verbose && /bin/sleep 86400" +stdout_logfile=${SUPERVISORD_LOG_PATH}/%(program_name)s_stdout.log +stdout_logfile_maxbytes=10MB +stdout_logfile_backups=10 +stderr_logfile=${SUPERVISORD_LOG_PATH}/%(program_name)s_stderr.log +stderr_logfile_maxbytes=10MB +stderr_logfile_backups=10 +startsecs=10 +stopsignal=QUIT +autorestart=true + +[program:${WEB2PY_NAME}_session_cleanup] +directory=${WEB2PY_PATH} +user=${APP_USER} +group=${APP_GROUP} +command=${WEB2PY_PATH}/bin/python web2py.py -S OZtree -M -R scripts/sessions2trash.py +stdout_logfile=${SUPERVISORD_LOG_PATH}/%(program_name)s_stdout.log +stdout_logfile_maxbytes=10MB +stdout_logfile_backups=10 +stderr_logfile=${SUPERVISORD_LOG_PATH}/%(program_name)s_stderr.log +stderr_logfile_maxbytes=10MB +stderr_logfile_backups=10 +startsecs=10 +stopsignal=QUIT +autorestart=true +EOF diff --git a/requirements.txt b/requirements.txt index 77947e063..d9d7db23d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ pymysql==1.1.0 +uwsgi==2.0.23 From c7be1572ec9f507bf11d2dcdda6c65510c2665be Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Sat, 20 Jan 2024 17:22:29 +0000 Subject: [PATCH 09/20] Gruntfile: Alter web2py.py to use venv #676 Update shebang at the top of web2py.py so it uses the venv by default. --- Gruntfile.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Gruntfile.js b/Gruntfile.js index 920702373..6376e844f 100755 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -41,6 +41,10 @@ module.exports = function (grunt) { '( [ -f applications/OZtree/private/appconfig.ini ] || { cp applications/OZtree/private/appconfig.ini.example applications/OZtree/private/appconfig.ini ; echo "****** edit private/appconfig.ini"; exit 1; } )', preferred_python3 + ' -m venv .', './bin/pip install -r applications/OZtree/requirements.txt', + // Make web2py.py use the venv + 'mv web2py.py web2py.py.o', + 'echo "#!' + venv_python + '" > web2py.py', + 'tail -n +2 web2py.py.o >> web2py.py', ].join(" && "), }, web2py_start_dev: { From 13a3db5b0668a12636a038b23305ed74f5e02449 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 12:03:53 +0000 Subject: [PATCH 10/20] README.markdown: Rework installation instructions Remove dependencies that are now part of tree-build, rework instructions to take into account new Grunt rules. Rework based on the --- README.markdown | 252 ++++++++++++++++-------------------------------- 1 file changed, 84 insertions(+), 168 deletions(-) diff --git a/README.markdown b/README.markdown index 6a340cbf7..405803588 100755 --- a/README.markdown +++ b/README.markdown @@ -10,218 +10,134 @@ There are two ways in which you can install OneZoom on a personal computer: full * *Full installation* creates an entire duplicate of the OneZoom website, which is built using the [web2py](http://web2py.com) framework. This creates a fully self-contained local system (apart from the picture files, which can be downloaded separately). This is the most reliable installation method, but requires you to install and run extra software packages, in particular [web2py](http://web2py.com) and a [MySQL](https://www.mysql.com) server. Since this can be quite complicated, the majority of this readme contains instructions for full installation. - ## Requirements and packages -For all installation methods, you will need to install node.js (and npm, the node package manager), and the webpack package. To compile the OneZoom javascript codebase automatically, you will then need to install grunt. To generate documentation or make a partial install, you will also need perl installed on your system. - -For full installation, you will additionally need to install web2py, and ensure that you have the programming language python installed on your system, which is what web2py uses. You will also need access to a database backend (e.g. mySQL running on your own computer, or on a remote server which you can administer). - -To create trees, you will need python and perl, along with a number of libraries, as listed below. - -### Required packages (you will need these even if you're not creating trees) -The OneZoom codebase uses the following software (licenses for each listed in braces). The first three are programming languages which may well already be installed on your computer. - -* [Python](https://www.python.org) (assumed version 3.7) with the following libraries installed: - * mysql-connector-python - * pymysql (needed even if not creating trees) - * piexif - * requests - * Dendropy - * (for functional testing) nose + js2py + selenium + e.g. chromedriver_installer -* [Perl](https://www.perl.org) with the following libraries installed - * File::ReadBackwards - * LWP::Simple - * JSON - * DBI - * Try::Tiny - * Text::CSV - * Image::ExifTool - * DBD::mysql -* [web2py](http://web2py.com) (LGPL license) -* [npm](https://www.npmjs.com/get-npm), part of node.js which, when run will install a large number of other packages including - * [grunt](https://gruntjs.com) (MIT licence): to automate creating the OneZoom website files - * [webpack](https://webpack.js.org) (MIT licence): to package the OneZoom javascript tree viewer into a library - * jsdoc-to-markdown (MIT licence): to produce documentation from source code -* [ImageMagick](https://www.imagemagick.org/script/index.php) (Apache 2.0) for processing thumbnails -* [curl](https://curl.haxx.se) (MIT-like licence) to download partial installs (`curl` is probably already installed on your computer) -* [UIkit 3](https://getuikit.com) (MIT licence) for the User Interface (this code is included in the OneZoom github repo, and does not need downloading) - -## Quick installation steps - -Before anything else, get the OZtree app from [github](https://github.com/OneZoom/OZtree) - see *"Downloading the OZtree app"*. You should also make sure you have node.js and the node package manager (npm), see *"Building the OneZoom tree viewer"* - - -### For a partial installation (less tested): - -1. Install the command-line version of `grunt` using `npm install -g grunt-cli`. You may need to have administrator privileges to do this. -2. From anywhere within the OZtree download, run `npm install` to install all the packages for automation. -3. Create a partial installation by running `grunt partial-install`. This downloads the "minlife" and "minlife_tour" pages from the central OneZoom website, modifies links within them, and places appropriately named html files into the `static` directory of your OZtree distribution. -4. Open e.g. `static/minlife.html` with a web browser of your choice (we recommend Chrome or Safari). Note that this file needs to stay within the static directory to work at all. You may also need to allow your browser to allow local files to be loaded via AJAX (i.e. disabling some local cross-origin checks). Different browsers do this in different ways: for example in Chrome you can start up your browser with the `--allow-file-access-from-files` option, and in Safari, you can choose "Disable Local File Restrictions" from the "Developer" menu. -5. Note that the normal `minlife.html` file will use local versions of data files and the javascript treeviewer, but will get API information form the OneZoom website, *and* also use the OneZoom website as the source for the html page which embeds the viewer. For developers only, who may wish to create a minlife version not only using modified javascript in the treeviewer but also with bespoke html, you can run `grunt partial-local-install`. This is much more effort since it requires you to set up a full installation (as below) before creating the minlife scripts, but once created, the files in `static` will be enough for other users to view (and test) your modifications. - -### For a full installation (recommended): - -1. Install a source code version of [web2py](http://www.web2py.com), placing your [OZtree repository](https://github.com/OneZoom/OZtree) within the web2py `applications` directory. -2. Install command-line software by running `npm install -g grunt-cli` (you may need to do all this with administrator privileges). -3. Run `npm install` from within the OZtree folder you moved in step 1. then run `grunt dev` (or `grunt prod` if in production mode) - see *"[Building the OneZoom tree viewer](#building-the-onezoom-tree-viewer)"*. -3. [Install](http://dev.mysql.com/downloads/mysql/) & start MySQL, then create a new database (see *"[Setting up the database backend](#setting-up-the-database-backend)"*) -4. Edit `private/appconfig.ini` file in `OZtree/private`, with `migrate=1` and with the appropriate database username and password. -5. Fire up a temporary web2py server and visit the main page to create the (empty) database tables - see *"[Starting and shutting down web2py](#starting-and-shutting-down-web2py)"* -6. Load up data into the tables: first create a user and assign it a 'manager' role in the `auth_` tables using the web2py database admin pages, then load the other tables using data from the original OneZoom site (e.g. sent to you via file transfer) - see *"[Filling the database](#filling-the-database)"*. -7. Optimise your installation: - * create indexes on the tables by running the SQL script in `OZtree/OZprivate/ServerScripts/SQL/create_db_indexes.sql`. You can do this, for example, by running `SOURCE /path/to/OZtree/OZprivate/ServerScripts/SQL/create_db_indexes.sql` within a mysql client. - * set `migrate=0` in appconfig.ini. +For any installation, OneZoom requires node & python (3.11). -## Downloading the OZtree app +#### Debian/Ubuntu -Download a copy of the OZtree application from GitHub at [https://github.com/OneZoom/OZtree](https://github.com/OneZoom/OZtree), either as a zip file (not recommended), or probably better (easier to update), by cloning the repository (e.g. using `git clone https://github.com/OneZoom/OZtree` or if you have [GitHub Desktop](https://desktop.github.com) installed, click "Open in Desktop" from the [OZtree repo](https://github.com/OneZoom/OZtree)). Make sure the git folder is called "OZtree" (this is the default when you clone the repo, but not if you download it as a zip file). - -For full installation, you will also need to download the source code version of web2py, either via git (https://github.com/web2py/web2py/) or simply from the download link at http://www.web2py.com/. You can then place the OZtree directory into the `applications` directory of the web2py folder. +``` +apt install nodejs npm +apt install python3 python3-dev python3-venv +``` +### Full installation -## Building the OneZoom tree viewer +In addition, for a full installation you also need nginx, supervisor & MySQL. -Compiling and creating the OneZoom explorer javascript code requires grunt to be installed. This compiles javascript code from multiple sources into a single file. You will need to install the node package manager, npm, then do +#### Debian/Ubuntu ``` -npm install -g grunt-cli -``` +apt install nginx -To install this is likely to require administrator privileges. Other required packages can then be installed from within the OZtree folder by simply typing the following (which may take a while to complete!) +apt install supervisor -``` -npm install +apt install lsb-release +wget https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb +dpkg -i mysql-apt-config_0.8.29-1_all.deb +apt update && apt install mysql-server +# NB: Select "Use Legacy Authentication Method (Retain MySQL 5.x Compatibility)" ``` -Once these are installed you can run grunt as follows (feel free to examine the configuration options which are stored in `Gruntfile.js` in the main OZtree directory): +#### Windows -#### Compile documentation -`grunt docs`: Use this command to generate a compiled documentation file. This will generate a large compiled markdown file in `OZprivate/rawJS/OZTreeModule/docs/_compiled.markdown`, which is best viewed once you have got web2py running, by pointing your browser to `dev/DOCS` (e.g. at `http://127.0.0.1:8000/dev/DOCS`). Note that viewing this page requires a working internet connection to get various formatting files) +On Windows we recommended downloading the MSI installer as it will make it easier to configure the new server during the installation +Once mysql is installed, you will need to set a root password, and create a database for web2py to use. See http://dev.mysql.com/doc/refman/5.7/en/default-privileges.html. +The mysqld program is responsible for running the new database just created. When this program is running, you can connect to the database. -#### In development mode: -`grunt dev`: This command bundles multiple js files into one. +## Installation -#### In production mode: -`grunt prod`: This command does three things. Firstly, it pre-compiles python code. Then it bundles multiple js files into one. Lastly, it minifies bundled js files. +If performing a full installation, you need to create a database for OneZoom to use: +``` +mysql -p +CREATE DATABASE OneZoom; +CREATE USER 'oz'@'localhost' IDENTIFIED BY 'passwd-you-should-change-this'; +GRANT ALL PRIVILEGES ON OneZoom . * TO 'oz'@'localhost'; +``` -## The server-side database +Firstly, you need to check out the web2py / OneZoom repositories, and run npm: -### Setting up the database backend +``` +# NB: The path should match the eventual hostname for this installation +export WEB2PY_PATH="/srv/.../onezoom.myhostname.org" +mkdir -p ${WEB2PY_PATH} +chown deploy:staff ${WEB2PY_PATH} +git clone https://github.com/web2py/web2py ${WEB2PY_PATH} --branch v2.27.1 +git clone https://github.com/OneZoom/OZtree.git ${WEB2PY_PATH}/applications/OZtree --branch production-next +cd ${WEB2PY_PATH}/applications/OZtree +npm ci --legacy-peer-deps +``` -The web2py instance requires a database to be running. We previously used sqllite, and code for interfacing with sqllite is still present in the codebase, but probably will not work, as we have switched to using mySQL. +Next, ``cp private/appconfig.ini.example private/appconfig.ini`` and edit to match your needs, taking care to: -So a major step when installing OneZoom is: +* Use the same database credentials as configured earlier. -1. Install a locally running copy of mySQL. Make sure the server is installed and not only the client - There are many ways to do this: see http://dev.mysql.com/downloads/mysql/. +Once done, either run - On Windows we recommended downloading the MSI installer as it will make it easier to configure the new server during the installation +* ``./node_modules/.bin/grunt prod`` for a production installation +* ``./node_modules/.bin/grunt dev`` for a development installation - Once mysql is installed, you will need to set a root password, and create a database for web2py to use. See http://dev.mysql.com/doc/refman/5.7/en/default-privileges.html. +### Partial installation - The mysqld program is responsible for running the new database just created. When this program is running, you can connect to the database. +If you'd like your installation to be a partial installation, run: -2. (optional) We find it useful to have a GUI interface to connect to the database and run SQL scripts, this can be used instead of using MySQL command line (similar to Windows command line) that is installed by default with MySQL. On Mac OS X we use the (excellent) http://www.sequelpro.com. On windows you could try http://www.mysql.com/products/workbench/ or https://www.quest.com/products/toad-for-mysql/ +``` +./node_modules/.bin/grunt partial-install +``` -3. Once mysql is installed, you will need to set a root password, and create a database for web2py to use. See http://dev.mysql.com/doc/refman/5.7/en/default-privileges.html. So once mysqld is running, you need to log in to the sql server with the root name and password (if you are using the command line, log in using `mysql -u root -p`), and issue the following SQL commands (the text after the `mysql>` prompt) to create a database for web2py to use: feel free to use a different *'passwd'*. +...and open `static/minlife.html` in a web browser. +Note that this file needs to stay within the static directory to work at all. +You may also need to allow your browser to allow local files to be loaded via AJAX (i.e. disabling some local cross-origin checks). +Different browsers do this in different ways: for example in Chrome you can start up your browser with the `--allow-file-access-from-files` option, and in Safari, you can choose "Disable Local File Restrictions" from the "Developer" menu. - ``` - mysql> create database OneZoom; - Query OK, 1 row affected (0.09 sec) - - mysql> CREATE USER 'oz'@'localhost' IDENTIFIED BY 'passwd'; - Query OK, 0 rows affected (0.19 sec) - - mysql> GRANT ALL PRIVILEGES ON OneZoom . * TO 'oz'@'localhost'; - Query OK, 0 rows affected (0.09 sec) - ``` +Note that the normal `minlife.html` file will use local versions of data files and the javascript treeviewer, +but will get API information form the OneZoom website, *and* also use the OneZoom website as the source for the html page which embeds the viewer. +For developers only, who may wish to create a minlife version not only using modified javascript in the treeviewer but also with bespoke html, you can run `grunt partial-local-install`. +This is much more effort since it requires you to set up a full installation (as below) before creating the minlife scripts, but once created, the files in `static` will be enough for other users to view (and test) your modifications. + +## Database setup / migation -The database is now up and running. We recommend that you do *not* load data into it immediately, but first create the tables by installing web2py with `migrate=1` set in the appconfig.ini file. After running an instance of web2py and visiting the new site, the correct table structure should be automatically created (see below). After that you can populate the data into the tables using downloaded files. +For full installations, you need to setup your database. -## Web2py installation +To create required tables, use web2py migrate: -Configuring the OneZoom application to use the database involves creating a file called 'appconfig.ini' in the `private` folder within the OZtree app, modified to use the username and password that you supplied above. A minimal appconfig.ini file to get the site working is +* Edit private/appconfig.ini, setting ``migrate=1`` +* ``./web2py-run tests/unit/test_modules_embed.py``, on startup this will create tables as necessary +* Edit private/appconfig.ini, migrate=0 + +A database dump should be used to add species information: ``` -; App configuration - -; db configuration - set migrate=0 once installed -[db] -uri = mysql://oz:passwd@127.0.0.1/OneZoom?set_encoding=utf8mb4 -migrate = 1 -pool_size = 1 - -; smtp address and credentials -[smtp] - -[twitter] - -[forms] -; form styling -formstyle = bootstrap3_inline -separator = - -[paypal] -url = https://www.sandbox.paypal.com - -[general] - -[images] -; * url_base: get thumbnail images from this source. If not -; defined, will default to the local version, but that -; means you will need to download >100,000 thumbnail images -; onto your machine. If you want to use the images on the -; OneZoom server, set this to `//images.onezoom.org/` -url_base = //images.onezoom.org/ - -[sponsorship] -; * allow. Should we allow the sponsorship page to be -; shown on this machine? Usually not allowed, except on the -; main OneZoom site (on museum displays people will not want -; to enter paypal etc details). -; * maintenance_mins: to enable maintenance mode (e.g. when -; switching beta and production to enable a new website version) -; set to the number of minutes you expect the site to be down -; note that if is_testing=False then you will probably need to -; restart the server for changes to this to take effect -; * reservation_time_limit_mins: how long to reserve a leaf while a -; user is looking at the sponsor page -; * unpaid_time_limit_mins: how long before a sponsored leaf becomes -; free again if we receive no payment notification -allow_sponsorship = 0 -maintenance_mins = 0 - -[api] -;If you want to get data from the Encyclopedia of Life, you need to put your own API key here. -;Fill it in using instructions at http://eol.org/info/api_overview -;eol_api_key = 11111111111 +# NB: OneZoom.dump.sql can be extracted from https://github.com/OneZoom/OZtree-docker +mysql -p +USE OneZoom +SOURCE /OneZoom.dump.sql ``` -In order to use web2py you need to have a python v3 installed, the latest version can be found at -[https://www.python.org/downloads/](https://www.python.org/downloads/) - -NB: on Windows, make sure that you add `python` (and ideally `python3`) to the windows path during install, or the commands below will not work +Finally, ensure indexes are created: -Assuming you have python version 3 installed, should now try starting web2py as follows. +``` +mysql -p +USE OneZoom +SOURCE ${WEB2PY_PATH}/applications/OZtree/OZprivate/ServerScripts/SQL/create_db_indexes.sql +``` -### Starting and shutting down web2py +## Database explorer -On the OneZoom main site, web2py is run using a combination of nginx and uwsgi. This is complete overkill if you just want to run a local copy of OneZoom for testing purposes. You can simply run a [temporary and basic web2py server using Python 3](http://www.web2py.com/books/default/chapter/29/03/overview#Startup). The simplest is to open a command-line prompt in the root web2py folder, and run the following (assuming the command `python3` is linked to something like Python 3.7) +(optional) We find it useful to have a GUI interface to connect to the database and run SQL scripts, this can be used instead of using MySQL command line (similar to Windows command line) that is installed by default with MySQL. On Mac OS X we use the (excellent) http://www.sequelpro.com. On windows you could try http://www.mysql.com/products/workbench/ or https://www.quest.com/products/toad-for-mysql/ -`python3 web2py.py -i 127.0.0.1 -p 8000 -a pass` +## Starting and shutting down web2py -* (NB: it is possible to run a secure OneZoom site over https. To try this using the basic web2py server, create a `.crt` and `.key` file, e.g. by running the following in the web2py root directory: `openssl req -newkey rsa:2048 -x509 -days 365 -nodes -keyout oz.key -out oz.crt`, then use them when running web2py, as in: `python3 web2py.py -c oz.crt -k oz.key -i 127.0.0.1 -p 8000 -a pass`) +On the OneZoom main site, web2py is run using a combination of nginx and uwsgi. This is complete overkill if you just want to run a local copy of OneZoom for testing purposes. You can simply run a [temporary and basic web2py server using Python 3](http://www.web2py.com/books/default/chapter/29/03/overview#Startup). The simplest is to open a command-line prompt in the root web2py folder, and run the following (assuming the command `python3` is linked to something like Python 3.7) +``` +./node_modules/.bin/grunt start-dev +``` When web2py is run, it will print instructions telling how to shut down the web2py server. For example, on Windows you might use `taskkill /f /pid XXXX`, where `XXXX` is the process id. -**If this is a new installation** you should now visit `http://127.0.0.1:8000/OZtree/default/` or `https://127.0.0.1:8000/OZtree/default/` to force web2py to create database tables. To load data into the tables, see "Loading Data", below. - -Once tables are created, and everything is working, you can set `migrate=0` in `private/appconfig.ini`. This will mean that web2py will not make any changes to table structures in the DB - -### Web2py folder structure +## Web2py folder structure #### Standard folders `databases` stores all the database structure. From ce35ac152f3bd180020fa25f8f714bf41d558419 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 14:32:26 +0000 Subject: [PATCH 11/20] Gruntfile: Use python3.10, not python3.11 #672 #699 --- Gruntfile.js | 2 +- README.markdown | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Gruntfile.js b/Gruntfile.js index 6376e844f..12b069569 100755 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -5,7 +5,7 @@ const path = require('path'); partial_install_site = "http://www.onezoom.org"; partial_local_install_site = "http://127.0.0.1:8000"; // if you are running a local installation -preferred_python3 = "python3.11"; // in case you have multiple python3 versions installed +preferred_python3 = "python3.10"; // in case you have multiple python3 versions installed web2py_py = path.join(path.dirname(path.dirname(process.cwd())), 'web2py.py'); venv_python = path.join(path.dirname(path.dirname(process.cwd())), 'bin/python'); diff --git a/README.markdown b/README.markdown index 405803588..2c2f46af5 100755 --- a/README.markdown +++ b/README.markdown @@ -39,6 +39,12 @@ apt update && apt install mysql-server # NB: Select "Use Legacy Authentication Method (Retain MySQL 5.x Compatibility)" ``` +#### FreeBSD + +``` +sudo pkg install nginx py39-supervisor lang/python310 +``` + #### Windows On Windows we recommended downloading the MSI installer as it will make it easier to configure the new server during the installation From c8ff21797f0fe3cfa7d3be7d68a5500f48f286e2 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 14:34:38 +0000 Subject: [PATCH 12/20] README: Don't mention temporary branch Ideally this would be separated out into instructions for a production instance, but at least not mentioning a temporary branch is good. --- README.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index 2c2f46af5..b19bcd773 100755 --- a/README.markdown +++ b/README.markdown @@ -70,7 +70,7 @@ export WEB2PY_PATH="/srv/.../onezoom.myhostname.org" mkdir -p ${WEB2PY_PATH} chown deploy:staff ${WEB2PY_PATH} git clone https://github.com/web2py/web2py ${WEB2PY_PATH} --branch v2.27.1 -git clone https://github.com/OneZoom/OZtree.git ${WEB2PY_PATH}/applications/OZtree --branch production-next +git clone https://github.com/OneZoom/OZtree.git ${WEB2PY_PATH}/applications/OZtree --branch production cd ${WEB2PY_PATH}/applications/OZtree npm ci --legacy-peer-deps ``` From 7350687c15e75e9e1db4fe345c524c7233ee9bf1 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 15:22:56 +0000 Subject: [PATCH 13/20] install-nginx: /var/db/acme/live/ isn't created OOTB An empty installation doesn't have /var/db/acme/live/, only /var/db/acme/. --- install-nginx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-nginx.sh b/install-nginx.sh index fe73320cd..0209769f6 100755 --- a/install-nginx.sh +++ b/install-nginx.sh @@ -12,7 +12,7 @@ WWW_IMAGES_SERVER_NAME="$(echo ${WWW_SERVER_NAME} | sed 's/^w*/images/')" # ima [ -d "/usr/local/etc/nginx" ] && NGINX_PATH="/usr/local/etc/nginx" mkdir -p "${NGINX_PATH}/conf.d/" NGINX_LOG_PATH="/var/log/nginx" -[ -d "/var/db/acme/live/" ] && NGINX_CERT_PATH="/var/db/acme/live/" +[ -d "/var/db/acme/" ] && NGINX_CERT_PATH="/var/db/acme/live/" [ -d "/var/lib/dehydrated/certs" ] && NGINX_CERT_PATH="/var/lib/dehydrated/certs" NGINX_DHPARAM_PATH="${NGINX_PATH}/dhparam.pem" [ -d "/var/db/acme/live" ] && NGINX_CHALLENGE_PATH="/var/db/acme/live" From c33b695f16cad6084614eb1f6c6c58e37d61e449 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 15:23:36 +0000 Subject: [PATCH 14/20] install-nginx: BSD doesn't know about ln -r --- install-nginx.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install-nginx.sh b/install-nginx.sh index 0209769f6..0b6319a5b 100755 --- a/install-nginx.sh +++ b/install-nginx.sh @@ -30,9 +30,9 @@ for SN in onezoom.org ${WWW_SERVER_NAME} ${WWW_IMAGES_SERVER_NAME}; do -out "${NGINX_CERT_PATH}/${SN}/fullchain-ss.pem" \ -subj "/CN=${SN}" \ -addext "subjectAltName = DNS:selfsigned.${SN}" - ln -rs ${NGINX_CERT_PATH}/${SN}/privkey-ss.pem ${NGINX_CERT_PATH}/${SN}/privkey.pem - ln -rs ${NGINX_CERT_PATH}/${SN}/fullchain-ss.pem ${NGINX_CERT_PATH}/${SN}/fullchain.pem - ln -rs ${NGINX_CERT_PATH}/${SN}/fullchain-ss.pem ${NGINX_CERT_PATH}/${SN}/chain.pem + ln -s ${NGINX_CERT_PATH}/${SN}/privkey-ss.pem ${NGINX_CERT_PATH}/${SN}/privkey.pem + ln -s ${NGINX_CERT_PATH}/${SN}/fullchain-ss.pem ${NGINX_CERT_PATH}/${SN}/fullchain.pem + ln -s ${NGINX_CERT_PATH}/${SN}/fullchain-ss.pem ${NGINX_CERT_PATH}/${SN}/chain.pem fi done From 7070d1ee82dca33105449dfbdb4456bb6270180a Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 15:24:42 +0000 Subject: [PATCH 15/20] install-nginx: Use NGINX_PATH when generating include config --- install-nginx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-nginx.sh b/install-nginx.sh index 0b6319a5b..27abe894c 100755 --- a/install-nginx.sh +++ b/install-nginx.sh @@ -77,7 +77,7 @@ http { server_names_hash_bucket_size 128; - include /etc/nginx/conf.d/*.conf; + include ${NGINX_PATH}/conf.d/*.conf; } EOF From 34241ae9982acb1f24d344d6cfd4225f202d6ef8 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 15:25:14 +0000 Subject: [PATCH 16/20] install-nginx: Remove v6 listeners Nginx falls over if v6 isn't available, which it isn't. --- install-nginx.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/install-nginx.sh b/install-nginx.sh index 27abe894c..cbca4e427 100755 --- a/install-nginx.sh +++ b/install-nginx.sh @@ -84,9 +84,7 @@ EOF cat < ${NGINX_PATH}/conf.d/onezoom.org.conf server { listen 80; - listen [::]:80; listen 443 ssl http2; - listen [::]:443 ssl http2; server_name server_name onezoom.org default; @@ -157,9 +155,7 @@ upstream uwsgi_${WEB2PY_NAME} { server { listen 80; - listen [::]:80; listen 443 ssl http2; - listen [::]:443 ssl http2; server_name ${WWW_SERVER_NAME}; @@ -222,9 +218,7 @@ cat < ${NGINX_PATH}/conf.d/${WWW_IMAGES_SERVER_NAME}.conf server { listen 80; - listen [::]:80; listen 443 ssl http2; - listen [::]:443 ssl http2; server_name ${WWW_IMAGES_SERVER_NAME}; server_tokens off; From 413b242d93bede6c9060f41c4edbdb4b0a665702 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 15:37:30 +0000 Subject: [PATCH 17/20] install-supervisord: BSD-compatible stat syntax --- install-supervisord.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-supervisord.sh b/install-supervisord.sh index 25388ad43..56c05a79d 100755 --- a/install-supervisord.sh +++ b/install-supervisord.sh @@ -3,7 +3,7 @@ set -eu PROJECT_PATH="${PROJECT_PATH-$(dirname "$(readlink -f "$0")")}" # The full project path WEB2PY_PATH="$(dirname $(dirname "$PROJECT_PATH"))" WEB2PY_NAME="${WEB2PY_NAME-$(basename ${WEB2PY_PATH})}" # Directory web2py lives in, will be unique per installation -DEPLOY_USER="$(stat -c '%U' $0)" +DEPLOY_USER="$(stat -c '%U' install-supervisord.sh 2>/dev/null || stat -f '%Su' install-supervisord.sh 2>/dev/null)" APP_USER="www" APP_GROUP="www" From 29588224a05ea42e09a7081c2433f3947de61b85 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 15:38:19 +0000 Subject: [PATCH 18/20] install-supervisord: Correct supervisord conf location on BSD There is no /usr/local/etc/supervisord, it's all in /usr/local/etc. --- install-supervisord.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-supervisord.sh b/install-supervisord.sh index 56c05a79d..edf6b9552 100755 --- a/install-supervisord.sh +++ b/install-supervisord.sh @@ -23,7 +23,7 @@ for DIR in "${PROJECT_PATH}/errors" \ done [ -d "/etc/supervisor" ] && SUPERVISORD_CONF_PATH="/etc/supervisor" -[ -d "/usr/local/etc/supervisord" ] && SUPERVISORD_CONF_PATH="/usr/local/etc/supervisord" +[ -e "/usr/local/etc/supervisord.conf.sample" ] && SUPERVISORD_CONF_PATH="/usr/local/etc" mkdir -p "${SUPERVISORD_CONF_PATH}/supervisord.conf.d/" mkdir -p "/var/run/supervisor" From 126f1ba2d66c1226b2c4b28c1476adb079ae8929 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 15:39:04 +0000 Subject: [PATCH 19/20] install-supervisord: Put script in debug mode It's comforting to see what it's doing, and makes error messages more intelligable. --- install-supervisord.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-supervisord.sh b/install-supervisord.sh index edf6b9552..40a404261 100755 --- a/install-supervisord.sh +++ b/install-supervisord.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -eu +set -eux PROJECT_PATH="${PROJECT_PATH-$(dirname "$(readlink -f "$0")")}" # The full project path WEB2PY_PATH="$(dirname $(dirname "$PROJECT_PATH"))" WEB2PY_NAME="${WEB2PY_NAME-$(basename ${WEB2PY_PATH})}" # Directory web2py lives in, will be unique per installation From b45dea23ad942b0d0d5c6c1d767765c8ff06aed7 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 15:50:17 +0000 Subject: [PATCH 20/20] README.markdown: Separate "production installation" section Move the production installation notes into their own section, mention the install scripts to be run as root. --- README.markdown | 60 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/README.markdown b/README.markdown index b19bcd773..90d724b73 100755 --- a/README.markdown +++ b/README.markdown @@ -12,7 +12,7 @@ There are two ways in which you can install OneZoom on a personal computer: full ## Requirements and packages -For any installation, OneZoom requires node & python (3.11). +For any installation, OneZoom requires node & python (3.10). #### Debian/Ubuntu @@ -21,17 +21,17 @@ apt install nodejs npm apt install python3 python3-dev python3-venv ``` -### Full installation +#### FreeBSD -In addition, for a full installation you also need nginx, supervisor & MySQL. +``` +sudo pkg install lang/python310 node18 npm-node18 +``` + +In addition, for a full installation you also need MySQL: #### Debian/Ubuntu ``` -apt install nginx - -apt install supervisor - apt install lsb-release wget https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb dpkg -i mysql-apt-config_0.8.29-1_all.deb @@ -39,12 +39,6 @@ apt update && apt install mysql-server # NB: Select "Use Legacy Authentication Method (Retain MySQL 5.x Compatibility)" ``` -#### FreeBSD - -``` -sudo pkg install nginx py39-supervisor lang/python310 -``` - #### Windows On Windows we recommended downloading the MSI installer as it will make it easier to configure the new server during the installation @@ -79,10 +73,9 @@ Next, ``cp private/appconfig.ini.example private/appconfig.ini`` and edit to mat * Use the same database credentials as configured earlier. -Once done, either run +Once done, run ``./node_modules/.bin/grunt dev`` for a development installation. -* ``./node_modules/.bin/grunt prod`` for a production installation -* ``./node_modules/.bin/grunt dev`` for a development installation +For production installation, see later. ### Partial installation @@ -129,6 +122,41 @@ USE OneZoom SOURCE ${WEB2PY_PATH}/applications/OZtree/OZprivate/ServerScripts/SQL/create_db_indexes.sql ``` +## Production installation + +For a production installation of OneZoom, you also need nginx & supervisor: + +#### Debian/Ubuntu + +``` +apt install nginx supervisor +``` + +#### FreeBSD + +``` +sudo pkg install nginx py39-supervisor lang/python310 +``` + +### Installation + +Run grunt to configure onezoom for production use: + +``` +cd ${WEB2PY_PATH}/applications/OZtree +npm ci --legacy-peer-deps +./node_modules/.bin/grunt prod +``` + +Then run the install scripts to set up nginx & supervisord: + +``` +sudo ./install-nginx.sh +sudo ./install-supervisord.sh +``` + +If everything works, restart both. + ## Database explorer (optional) We find it useful to have a GUI interface to connect to the database and run SQL scripts, this can be used instead of using MySQL command line (similar to Windows command line) that is installed by default with MySQL. On Mac OS X we use the (excellent) http://www.sequelpro.com. On windows you could try http://www.mysql.com/products/workbench/ or https://www.quest.com/products/toad-for-mysql/