diff --git a/.gitignore b/.gitignore index c4b9d2495..4a7e9f951 100644 --- a/.gitignore +++ b/.gitignore @@ -20,5 +20,8 @@ GITHUB_CLIENT_SECRET GITHUB_APP_PRIVATE_KEY_PEM smasher/lib/json-simple-1.1.1.jar deploy/setup/CONFIG +env +web2py +restart.sh webapp/static/statistics/ lint/ diff --git a/LICENSE.txt b/LICENSE.txt index 7c5650a3f..51e069afd 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,7 +1,7 @@ -Copyright (c) 2013, Jonathan Rees -Copyright (c) 2013, Mark Holder -Copyright (c) 2013, Jim Allman -Copyright (c) 2013, Stephen Smith +Copyright (c) 2013-2016, Jonathan Rees +Copyright (c) 2013-2016, Mark Holder +Copyright (c) 2013-2016, Jim Allman +Copyright (c) 2013-2016, Stephen Smith All rights reserved. diff --git a/README.md b/README.md index fc63c97f0..784f1d965 100644 --- a/README.md +++ b/README.md @@ -1,101 +1,89 @@ -opentree -======== +# opentree -This is the repository for the Open Tree of Life web applications, one of many subsystems making up the Open Tree of Life project code. +This is the repository for the Open Tree of Life web applications, one of many subsystems making up +the Open Tree of Life project code. +For Open Tree of Life documentation, see +[the germinator repository's wiki](https://github.com/OpenTreeOfLife/germinator/wiki). +The 'deployment system' and web API documentation sources that formerly resided in this +repository now live in the [germinator repository](https://github.com/OpenTreeOfLife/germinator). +The following instructions have not been reviewed in a long time. +For local installation a better place to start might be +[this wiki page](https://github.com/OpenTreeOfLife/opentree/wiki/Installing-a-local-curator-and-tree-browser-test-server). -For Open Tree of Life documentation, see [the germinator repository's wiki](https://github.com/OpenTreeOfLife/germinator/wiki). The 'deployment system' and web API documentation sources that formerly resided in this repository now live in the [germinator repository](https://github.com/OpenTreeOfLife/germinator). - -The following instructions have not been reviewed in a long time. For local installation a better place to start might be [this wiki page](https://github.com/OpenTreeOfLife/opentree/wiki/Installing-a-local-curator-and-tree-browser-test-server). - -Installation -============ -See the phylografter instructions for -more details about using web2py. +## Installation We strongly recommend using a virtual environment to manage the version of -Python and installed modules. We're currently running opentree with Python -v2.7.3. Newer versions of python2.7 should work, but **NOTE that web2py is not +Python and installed modules. +We're currently running opentree with Python +v2.7.3. +Newer versions of python2.7 should work, but **NOTE that web2py is not compatible with Python 3**. +The final invocation to create your virtualenv should look something like: -If necessary, compile Python2.7 and use it when making your virtualenv. You -should be able to safely install multiple versions of python using your -preferred package manager, or by configuring Python2.7 with the --prefix -option and 'make altinstall'. - -So the final invocation to create your virtualenv should look something like: -``` -$ virtualenv --python=/usr/bin/python2.7 --distribute -``` + $ virtualenv --python=(which python2.7) --distribute env + $ source env/bin/activate -Or, if you're using virtualenvwrapper (http://virtualenvwrapper.readthedocs.org/en/latest/index.html): -``` -$ mkvirtualenv --python=python2.7 --no-site-packages --distribute opentree -``` The included **requirements.txt** file lists known-good versions of all the required python modules for opentree, plus a few convenience modules. To [install these modules using pip](http://www.pip-installer.org/en/latest/cookbook.html#requirements-files), -
-pip install -r requirements.txt
-
+ pip install -r requirements.txt + +### install web2py and link to applications The contents of the webapp subdirectory are a web2py application. Make a symbolic link called "opentree" in a web2py/applications directory to the webapp directory. You should be able to launch web2py and see the app running at http://127.0.0.1:8000/opentree/ -There is now a second web2py app for the curation tool, which will also need a -symlink. This will be available at http://127.0.0.1:8000/curator/ + wget --no-verbose -O web2py_2.8.2_src.zip \ + https://github.com/web2py/web2py/archive/R-2.8.2.zip + unzip web2py_2.8.2_src.zip + mv web2py_2.8.2_src web2py + cd web2py/applications + ln -s ../../webapp opentree + cd - + cp -p oauth20_account.py web2py/gluon/contrib/login_methods/ + cp -p rewrite.py web2py/gluon/ + cp -p custom_import.py web2py/gluon/ + cp -p SITE.routes.py web2py/routes.py -Briefly: -1. Download and unpack the source code version of web2py from -http://www.web2py.com/examples/default/download MTH used version 2.4.2 of web2py +Optionally, you can install a second web2py app for the curation tool, which will also need a +symlink. This will be available at http://127.0.0.1:8000/curator/ - NOTE: This version of web2py includes basic support for OAuth 2.0, but it needs - a minor patch to support for login via the GitHub API v3. (The curation app - uses GitHub for its datastore and attribution. The tree browser also uses it - for its issue tracker, with optional authentication for convenience.) Replace - this web2py file with a modified version in the same folder as this README: - 
-   {web2py-2.4.4}/gluon/contrib/login_methods/oauth20_account.py
-
+ cd web2py/applications + ln -s ../../curator curator + cd - -2. Create the sym links for the main web app and the study curation tool. +## Configuring the application +Copy and tweak the template `config` files to provide the variables that will +be available during execution of the web applications: - 
-   cd web2py/applications
-   ln -s /full/path/to/opentree/webapp opentree
-   ln -s /full/path/to/opentree/curator curator
-
+ for app in webapp curator ; do + cp ${app}/private/config.example ${app}/private/config + done -3. Customize web2py's site-wide routing behavior using "SITE.routes.py" +Edit each `${app}/private/config` file to use the API servers and properties that you + want to use for debugging purposes. - 
-   # return to main web2py directory
-   cd ..  
-   cp /full/path/to/opentree/SITE.routes.py routes.py
-
- - This routing file works in tandem with the opentree app router and lets us have - proper URLs with hyphens instead of underscores. -4. Launch web2py +## Launch web2py for debugging - 
-   cd /full/path/to/web2py
-   python web2py.py --nogui -a '<recycle>'
-
+ cd web2py + python web2py.py --nogui -a '' Where the -a flag is allowing you to reuse the previous admin password that you used with this instance of web2py. +## For an instance that allows logging in + **To test with login and proper domain name**, modify your test system's `/etc/hosts` file (or equivalent) to resolve the domain `devtree.opentreeoflife.org` to localhost (127.0.0.1). Then run web2py on (privileged) port 80 like so: 
-   cd /full/path/to/web2py
+   cd web2py
    sudo python web2py.py --nogui -p 80 -a '<recycle>'
diff --git a/curator/controllers/collection.py b/curator/controllers/collection.py index ed13dc8cc..f62ccc22e 100644 --- a/curator/controllers/collection.py +++ b/curator/controllers/collection.py @@ -14,7 +14,7 @@ ######################################################################### from applications.opentree.modules.opentreewebapputil import( - get_opentree_services_method_urls, + get_opentree_api_endpoints, fetch_current_TNRS_context_names, fetch_trees_queued_for_synthesis, get_maintenance_info) @@ -27,7 +27,7 @@ def index(): Show list searchable/filtered list of all collections (default filter = My Collections, if logged in?) """ - view_dict = get_opentree_services_method_urls(request) + view_dict = get_opentree_api_endpoints(request) view_dict['maintenance_info'] = get_maintenance_info(request) if auth.is_logged_in(): # user is logged in, filter to their own collections by default? @@ -46,7 +46,7 @@ def view(): ? OR can this include work-in-progress from a personal branch? """ response.view = 'collection/edit.html' - view_dict = get_opentree_services_method_urls(request) + view_dict = get_opentree_api_endpoints(request) view_dict['maintenance_info'] = get_maintenance_info(request) #view_dict['taxonSearchContextNames'] = fetch_current_TNRS_context_names(request) view_dict['collectionID'] = request.args[0] +'/'+ request.args[1] @@ -65,7 +65,7 @@ def create(): if maintenance_info.get('maintenance_in_progress', False): redirect(URL('curator', 'default', 'index', vars={"maintenance_notice":"true"})) pass - view_dict = get_opentree_services_method_urls(request) + view_dict = get_opentree_api_endpoints(request) view_dict['message'] = "collection/create" return view_dict """ @@ -80,7 +80,7 @@ def edit(): args=request.args)) # Fetch a fresh list of search contexts for TNRS? see working example in # the header search of the main opentree webapp - view_dict = get_opentree_services_method_urls(request) + view_dict = get_opentree_api_endpoints(request) view_dict['taxonSearchContextNames'] = fetch_current_TNRS_context_names(request) view_dict['treesQueuedForSynthesis'] = fetch_trees_queued_for_synthesis(request) view_dict['collectionID'] = request.args[0] +'/'+ request.args[1] @@ -108,6 +108,17 @@ def load(): def store(): return dict(message="collection/store") +def synthesis_dashboard(): + """ + Allow any visitor to view (read-only!) a queue of recent custom-synthesis runs + """ + response.view = 'collection/synthesis_dashboard.html' + view_dict = get_opentree_services_method_urls(request) + view_dict['maintenance_info'] = get_maintenance_info(request) + view_dict['taxonSearchContextNames'] = fetch_current_TNRS_context_names(request) + view_dict['userCanEdit'] = auth.is_logged_in() and True or False + return view_dict + """ TODO: Adapt this for current collection status, based on new APIs """ def _get_latest_synthesis_details_for_collection_id( collection_id ): # Fetch the last SHA for this collection that was used in the latest @@ -118,7 +129,7 @@ def _get_latest_synthesis_details_for_collection_id( collection_id ): import json import requests - method_dict = get_opentree_services_method_urls(request) + method_dict = get_opentree_api_endpoints(request) # fetch a list of all studies and collections that contribute to synthesis # TODO: Request that these fields be added @@ -137,15 +148,6 @@ def _get_latest_synthesis_details_for_collection_id( collection_id ): # Draft code is based on schema proposed in # https://github.com/OpenTreeOfLife/phylesystem-api/issues/228 - # fetch the full source list, then look for this study and its trees - commit_SHA_in_synthesis = None - # if key (collection ID, e.g. "opentreeoflife/default") matches, read its details - for c_id, collection_details in source_dict.items(): - if c_id == collection_id: - # this is the collection we're interested in! - commit_SHA_in_synthesis = collection_details['git_sha'] - return commit_SHA_in_synthesis # TODO: return more information? - # fetch the full source list, then look for this collection and its SHA # if key (collection ID, e.g. "opentreeoflife/default") matches, read its details for c_id, collection_details in source_dict.items(): @@ -155,4 +157,5 @@ def _get_latest_synthesis_details_for_collection_id( collection_id ): return None except Exception, e: # throw 403 or 500 or just leave it - raise HTTP(500, T('Unable to retrieve latest synthesis details for collection {u}'.format(u=collection))) + ##raise HTTP(500, T('Unable to retrieve latest synthesis details for collection {u}'.format(u=collection_id))) + raise HTTP(500, T('Unable to retrieve latest synthesis details for collection {u}:\n\n{e}'.format(u=collection_id, e=e))) diff --git a/curator/controllers/default.py b/curator/controllers/default.py index df7d7b3d8..6907362b6 100644 --- a/curator/controllers/default.py +++ b/curator/controllers/default.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- from applications.opentree.modules.opentreewebapputil import( - get_opentree_services_method_urls, + get_opentree_api_endpoints, extract_nexson_from_http_call, fetch_github_app_auth_token, get_maintenance_info) @@ -29,7 +29,7 @@ def index(): a logged-in user. """ #response.flash = T("Welcome to web2py!") - view_dict = get_opentree_services_method_urls(request) + view_dict = get_opentree_api_endpoints(request) view_dict['maintenance_info'] = get_maintenance_info(request) if False: ## auth.is_logged_in(): @@ -45,12 +45,12 @@ def collections(): TODO: move to collection/index? """ - view_dict = get_opentree_services_method_urls(request) + view_dict = get_opentree_api_endpoints(request) view_dict['maintenance_info'] = get_maintenance_info(request) return view_dict def error(): - view_dict = get_opentree_services_method_urls(request) + view_dict = get_opentree_api_endpoints(request) return view_dict @auth.requires_login() @@ -78,7 +78,7 @@ def profile(): shows a personalized profile for any user (default = the current logged-in user) http://..../{app}/default/profile/[username] """ - view_dict = get_opentree_services_method_urls(request) + view_dict = get_opentree_api_endpoints(request) view_dict['maintenance_info'] = get_maintenance_info(request) # if the URL has a [username], try to load their information @@ -192,7 +192,7 @@ def _get_opentree_activity( userid=None, username=None ): 'added_collections':[], 'curated_collections':[] } - method_dict = get_opentree_services_method_urls(request) + method_dict = get_opentree_api_endpoints(request) # Use GitHub API to gather comments from this user, as shown in # https://github.com/OpenTreeOfLife/feedback/issues/created_by/jimallman @@ -470,6 +470,7 @@ def to_nexson(): _LOG = get_logger(request, 'to_nexson') if request.env.request_method == 'OPTIONS': raise HTTP(200, T('Preflight approved!')) + orig_args = {} is_upload = False # several of our NexSON use "uploadid" instead of "uploadId" so we should accept either @@ -622,7 +623,7 @@ def to_nexson(): try: assert(os.path.exists(exe_path)) except: - response.view = 'generic.json'; return {'hb':exe_path} + #response.view = 'generic.json'; return {'hb':exe_path} _LOG.warn("Could not find the 2nexml executable") raise HTTP(501, T("Server is misconfigured for 2nexml conversion")) invoc = [exe_path, '-f{f}'.format(f=inp_format), ] diff --git a/curator/controllers/study.py b/curator/controllers/study.py index 1e1a9627c..47ce44069 100644 --- a/curator/controllers/study.py +++ b/curator/controllers/study.py @@ -12,7 +12,7 @@ ######################################################################### from applications.opentree.modules.opentreewebapputil import( - get_opentree_services_method_urls, + get_opentree_api_endpoints, fetch_current_TNRS_context_names, fetch_trees_queued_for_synthesis, get_maintenance_info) @@ -25,7 +25,7 @@ def index(): Show list searchable/filtered list of all studies (default filter = My Studies, if logged in?) """ - view_dict = get_opentree_services_method_urls(request) + view_dict = get_opentree_api_endpoints(request) if auth.is_logged_in(): # user is logged in, filter to their own studies by default? @@ -44,7 +44,7 @@ def view(): ? OR can this include work-in-progress from a personal branch? """ response.view = 'study/edit.html' - view_dict = get_opentree_services_method_urls(request) + view_dict = get_opentree_api_endpoints(request) view_dict['maintenance_info'] = get_maintenance_info(request) #view_dict['taxonSearchContextNames'] = fetch_current_TNRS_context_names(request) view_dict['studyID'] = request.args[0] @@ -61,7 +61,7 @@ def create(): if maintenance_info.get('maintenance_in_progress', False): redirect(URL('curator', 'default', 'index', vars={"maintenance_notice":"true"})) pass - view_dict = get_opentree_services_method_urls(request) + view_dict = get_opentree_api_endpoints(request) view_dict['message'] = "study/create" return view_dict @@ -76,7 +76,7 @@ def edit(): args=request.args)) # Fetch a fresh list of search contexts for TNRS? see working example in # the header search of the main opentree webapp - view_dict = get_opentree_services_method_urls(request) + view_dict = get_opentree_api_endpoints(request) view_dict['taxonSearchContextNames'] = fetch_current_TNRS_context_names(request) view_dict['treesQueuedForSynthesis'] = fetch_trees_queued_for_synthesis(request) view_dict['studyID'] = request.args[0] @@ -93,7 +93,7 @@ def _get_latest_synthesis_details_for_study_id( study_id ): import json import requests - method_dict = get_opentree_services_method_urls(request) + method_dict = get_opentree_api_endpoints(request) # fetch a list of all studies that contribute to synthesis fetch_url = method_dict['getSynthesisSourceList_url'] diff --git a/curator/controllers/tnrs.py b/curator/controllers/tnrs.py index 450edadd1..96888a812 100644 --- a/curator/controllers/tnrs.py +++ b/curator/controllers/tnrs.py @@ -6,7 +6,7 @@ ######################################################################### from applications.opentree.modules.opentreewebapputil import( - get_opentree_services_method_urls, + get_opentree_api_endpoints, fetch_current_TNRS_context_names, get_maintenance_info) @@ -19,7 +19,7 @@ def index(): """ response.view = 'tnrs.html' - view_dict = get_opentree_services_method_urls(request) + view_dict = get_opentree_api_endpoints(request) #view_dict['message'] = "This would appear at bottom of page.." view_dict['maintenance_info'] = get_maintenance_info(request) view_dict['taxonSearchContextNames'] = fetch_current_TNRS_context_names(request) diff --git a/curator/private/config.example b/curator/private/config.example index 96f3f0982..fc58b34d5 100644 --- a/curator/private/config.example +++ b/curator/private/config.example @@ -34,6 +34,9 @@ github_redirect_uri = YOUR_REDIRECT_URI_HERE # (the installation ID is in the URL of the Configure button here) github_app_installation_id = YOUR_APP_INSTALLATION_ID_HERE +# +# TODO: Revise both API sections below to match our Ansible stuff, or delete this file entirely! +# # List public-facing base URL for supporting data services # (NOTE that these are used by both server- and client-side code) [domains] @@ -60,6 +63,9 @@ getContextForNames_url = {taxomachine_domain}/v3/tnrs/infer_context getSynthesisSourceList_url = {CACHED_treemachine_domain}/v3/tree_of_life/about getTaxonomicMRCAForNodes_url = {taxomachine_domain}/v3/taxonomy/mrca getDraftTreeMRCAForNodes_url = {treemachine_domain}/v3/tree_of_life/mrca +findAllSynthesisRuns_url = https://ot38.opentreeoflife.org/v3/tree_of_life/list_custom_built_trees +requestNewSynthesisRun_url = https://ot38.opentreeoflife.org/v3/tree_of_life/build_tree +# TODO: This should start {treemachine_domain} OR {CACHED_treemachine_domain} findAllStudies_url = {CACHED_oti_domain}/v3/studies/find_studies # TODO: Can we use CACHED_oti_domain for this? singlePropertySearchForStudies_url = {oti_domain}/v3/studies/find_studies diff --git a/curator/routes.py b/curator/routes.py new file mode 100644 index 000000000..8bfaeb14a --- /dev/null +++ b/curator/routes.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +# adapted from router.example.py + +# NOTE that this requires a parametric router in the web2py root directory. +# Let's keep all the important stuff here, and just copy a minimal router +# (SITE.routes.py) into the site root. + +# NOTE that this (app-specific) routes.py file mainly defines a router by the +# same name. More general settings must be done in the main routes.py alongside +# the web2py/applications/ directory +# root_static (for favicon.ico, robots.txt, etc) +# routes_onerror (defines error pages per app, per error code, or defaults) +# domain (maps domain names and ports to particular app) +# See SITE.routes.py for recommended settings. + +routers = dict( + curator=dict( + # convert dashes (hyphens) in URLs to underscores in web2py controller+action names + map_hyphen=True, + ), +) + +# see router.example.py for (many) more options! diff --git a/curator/static/css/default.css b/curator/static/css/default.css index f182447dd..c975d5871 100644 --- a/curator/static/css/default.css +++ b/curator/static/css/default.css @@ -870,7 +870,13 @@ tr.after-shims th { position: relative; top: -8px; } -.collection-move-panel { + +.form-horizontal.metadata-readonly .control-group { + margin-bottom: 12px; +} + +.collection-move-panel, +.synthrun-move-panel { position: absolute; margin-left: 2px; z-index: 1; @@ -880,7 +886,8 @@ tr.after-shims th { -moz-box-shadow: 1px 2px 4px rgba(0, 0, 0, 0.5); box-shadow: 1px 2px 4px rgba(0, 0, 0, 0.5); } -.collection-move-panel button { +.collection-move-panel button, +.synthrun-move-panel button { opacity: 1.0; } @@ -1014,3 +1021,11 @@ form.skip-label * { form.skip-label *.non-fading { opacity: 1.0; } +.loading-message { + display: inline-block; + position: relative; + top: -6px; + left: 8px; + color: #999; + font-style: italic; +} diff --git a/curator/static/js/bootstrap-tagsinput.js b/curator/static/js/bootstrap-tagsinput.js new file mode 100644 index 000000000..4c97aceea --- /dev/null +++ b/curator/static/js/bootstrap-tagsinput.js @@ -0,0 +1,503 @@ +(function ($) { + "use strict"; + + var defaultOptions = { + tagClass: function(item) { + return 'label label-info'; + }, + itemValue: function(item) { + return item ? item.toString() : item; + }, + itemText: function(item) { + return this.itemValue(item); + }, + freeInput: true, + maxTags: undefined, + confirmKeys: [13], + onTagExists: function(item, $tag) { + $tag.hide().fadeIn(); + } + }; + + /** + * Constructor function + */ + function TagsInput(element, options) { + this.itemsArray = []; + + this.$element = $(element); + this.$element.hide(); + + this.isSelect = (element.tagName === 'SELECT'); + this.multiple = (this.isSelect && element.hasAttribute('multiple')); + this.objectItems = options && options.itemValue; + this.placeholderText = element.hasAttribute('placeholder') ? this.$element.attr('placeholder') : ''; + this.inputSize = Math.max(1, this.placeholderText.length); + + this.$container = $('
'); + this.$input = $('').appendTo(this.$container); + + this.$element.after(this.$container); + + this.build(options); + } + + TagsInput.prototype = { + constructor: TagsInput, + + /** + * Adds the given item as a new tag. Pass true to dontPushVal to prevent + * updating the elements val() + */ + add: function(item, dontPushVal) { + var self = this; + + if (self.options.maxTags && self.itemsArray.length >= self.options.maxTags) + return; + + // Ignore falsey values, except false + if (item !== false && !item) + return; + + // Throw an error when trying to add an object while the itemValue option was not set + if (typeof item === "object" && !self.objectItems) + throw("Can't add objects when itemValue option is not set"); + + // Ignore strings only containg whitespace + if (item.toString().match(/^\s*$/)) + return; + + // If SELECT but not multiple, remove current tag + if (self.isSelect && !self.multiple && self.itemsArray.length > 0) + self.remove(self.itemsArray[0]); + + if (typeof item === "string" && this.$element[0].tagName === 'INPUT') { + var items = item.split(','); + if (items.length > 1) { + for (var i = 0; i < items.length; i++) { + this.add(items[i], true); + } + + if (!dontPushVal) + self.pushVal(); + return; + } + } + + var itemValue = self.options.itemValue(item), + itemText = self.options.itemText(item), + tagClass = self.options.tagClass(item); + + // Ignore items allready added + var existing = $.grep(self.itemsArray, function(item) { return self.options.itemValue(item) === itemValue; } )[0]; + if (existing) { + // Invoke onTagExists + if (self.options.onTagExists) { + var $existingTag = $(".tag", self.$container).filter(function() { return $(this).data("item") === existing; }); + self.options.onTagExists(item, $existingTag); + } + return; + } + + // register item in internal array and map + self.itemsArray.push(item); + + // add a tag element + var $tag = $('' + htmlEncode(itemText) + ''); + $tag.data('item', item); + self.findInputWrapper().before($tag); + $tag.after(' '); + + // add