diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..4acb621b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +.git +.idea +properties +OLS_logo_and_family +*.rtf +*.iml +*.md +!ols-web/src/main/resources/application.properties +!ols-web/target/ols-boot.war +!ols-apps/ols-config-importer/target/ols-config-importer.jar +!ols-apps/ols-loading-app/target/ols-indexer.jar +!ols-solr/src/main/solr-5-config +data diff --git a/.gitignore b/.gitignore index 81fb8a7f..0384ebc4 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,11 @@ ols-loader/src/test/resources/tmp ols-solr/src/main/resources/solr-conf/ontology/data ols-mongo/src/main/resources/mongodb ols-web/src/main/resources/static/documents/* -properties \ No newline at end of file +properties +.classpath +.settings +.project +*.log +new_ols.css +new_ols.css.map +.factorypath diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..379abd7a --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,25 @@ + +build: + image: docker:18-git + stage: build + only: + - dev + - test + - stable + services: + - docker:18-dind + script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker build -t ${OLS_IMAGE_PREFIX}/ols-config-importer:$CI_COMMIT_REF_NAME-$CI_BUILD_ID -f ./ols-apps/ols-config-importer/Dockerfile . + - docker build -t ${OLS_IMAGE_PREFIX}/ols-indexer:$CI_COMMIT_REF_NAME-$CI_BUILD_ID -f ./ols-apps/ols-indexer/Dockerfile . + - docker build -t ${OLS_IMAGE_PREFIX}/ols-web:$CI_COMMIT_REF_NAME-$CI_BUILD_ID -f ./ols-web/Dockerfile . + - docker tag ${OLS_IMAGE_PREFIX}/ols-config-importer:$CI_COMMIT_REF_NAME-$CI_BUILD_ID ${OLS_IMAGE_PREFIX}/ols-config-importer:$CI_COMMIT_REF_NAME + - docker tag ${OLS_IMAGE_PREFIX}/ols-indexer:$CI_COMMIT_REF_NAME-$CI_BUILD_ID ${OLS_IMAGE_PREFIX}/ols-indexer:$CI_COMMIT_REF_NAME + - docker tag ${OLS_IMAGE_PREFIX}/ols-web:$CI_COMMIT_REF_NAME-$CI_BUILD_ID ${OLS_IMAGE_PREFIX}/ols-web:$CI_COMMIT_REF_NAME + - docker push ${OLS_IMAGE_PREFIX}/ols-config-importer:$CI_COMMIT_REF_NAME-$CI_BUILD_ID + - docker push ${OLS_IMAGE_PREFIX}/ols-config-importer:$CI_COMMIT_REF_NAME + - docker push ${OLS_IMAGE_PREFIX}/ols-indexer:$CI_COMMIT_REF_NAME-$CI_BUILD_ID + - docker push ${OLS_IMAGE_PREFIX}/ols-indexer:$CI_COMMIT_REF_NAME + - docker push ${OLS_IMAGE_PREFIX}/ols-web:$CI_COMMIT_REF_NAME-$CI_BUILD_ID + - docker push ${OLS_IMAGE_PREFIX}/ols-web:$CI_COMMIT_REF_NAME + diff --git a/OLS-Architecture.png b/OLS-Architecture.png new file mode 100644 index 00000000..8c4b68de Binary files /dev/null and b/OLS-Architecture.png differ diff --git a/README.md b/README.md index 9bc3f320..bdbd70a0 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,139 @@ # OLS -Ontology Lookup Service from SPOT at EBI. + +Ontology Lookup Service from SPOT at EBI. * OLS is currently live at the EBI here http://www.ebi.ac.uk/ols * A REST API for OLS is described here http://www.ebi.ac.uk/ols/docs/api -* Instruction on how to build a local OLS installation are here http://www.ebi.ac.uk/ols/docs/installation-guide - * Run OLS with docker here https://github.com/MaastrichtUniversity/ols-docker -* Further OLS documentation can be found here http://www.ebi.ac.uk/ols/docs +* Instructions on how to build a local OLS installation are here + http://www.ebi.ac.uk/ols/docs/installation-guide +* Run OLS with docker here + https://github.com/MaastrichtUniversity/ols-docker +* Further OLS documentation can be found here + http://www.ebi.ac.uk/ols/docs ## Overview -This is the entire codebase for the EBI OLS. OLS has been developed around two key ontology indexes that can be built and used - independently from the core website. We provide service to build a SOLR index and and a Neo4j index. The SOLR index is used to provide text based queries over the ontologies while the neo4j index is used to query the ontology structure and is the primary driver of the OLS REST API. +![OLS Architecture](OLS-Architecture.png) + +This is the entire codebase for the EBI OLS. OLS has been developed +around two key ontology indexes that can be built and used independently +from the core website. We provide services to build a Solr index and a +Neo4j index. The Solr index is used to provide text-based queries over +the ontologies while the Neo4j index is used to query the ontology +structure and is the primary driver of the OLS REST API. + +OLS has been developed with the Spring Data and Spring Boot framework. +You can build this project with Maven and the following Spring Boot +applications will be available to run. - OLS has been developed with the Spring data and Spring boot framework. You can build this project with Maven and the following Spring Boot applications will be available to run. +All of the apps are available under the ols-apps module. + +* [ols-apps/ols-solr-app](ols-apps/ols-solr-app) - Spring Boot + application for building a Solr index for one or more ontologies. + Requires access to a Solr server. +* [ols-apps/ols-neo4j-app](ols-apps/ols-neo4j-app) - Spring Boot + application for building a Neo4j index for one or more ontologies. + Builds an embedded Neo4j database. You can run a Neo4j server that + uses the generated Neo4j database. + +To run a complete local OLS installation you will need a MongoDB +database. This is a lightweight database that is used to store all the +ontology configuration and application state information. See here for +more information http://www.ebi.ac.uk/ols/docs/installation-guide + +* [ols-apps/ols-config-importer](ols-apps/ols-config-importer) - Spring + Boot application for loading config files into the MongoDB database. + This includes support for reading config files specified using the OBO + Foundry YAML format. +* [ols-apps/ols-indexer](ols-apps/ols-indexer) - Spring Boot + application for building the complete OLS indexes. This app fetches + ontologies specified in the config files, checks whether they have + changed from a previous download, and if they have changed, will + create all the necessary Solr and Neo4j indexes. +* [ols-web](ols-web) - This contains the WAR file that can be deployed + in Tomcat to launch the OLS website and REST API. It depends on + [ols-term-type-treeview] + (https://github.com/EBISPOT/ols-term-type-treeview) and + [ols-tabbed-term-treeview] + (https://github.com/EBISPOT/ols-tabbed-term-treeview). + + +## Deploying with Docker + +The preferred method of deployment for OLS is using Docker. First, create the +necessary volumes: + + docker volume create --name=ols-neo4j-data + docker volume create --name=ols-mongo-data + docker volume create --name=ols-solr-data + docker volume create --name=ols-downloads + +Then, start solr and mongodb only: + + docker-compose up -d solr mongo + +Then, adjust the configuration YAML files in the `config` directory as required, +and load the configuration into the Mongo database using the config loader: + + docker run --net=host -v $(pwd)/config:/config ebispot/ols-config-importer:stable + +Then, run the indexer: + + docker run --net=host -v ols-neo4j-data:/mnt/neo4j -v ols-downloads:/mnt/downloads ebispot/ols-indexer:stable + +Finally, start the OLS webserver: + + docker-compose up -d ols-web + +You should now be able to access a populated OLS instance at `http://localhost:8080`. + + +### Building the Docker images manually + +Rather than using the images from Docker Hub, the Docker images can also be +built using the Dockerfiles in this repository. + + docker build -f ols-apps/ols-config-importer/Dockerfile -t ols-config-importer . + docker build -f ols-apps/ols-indexer/Dockerfile -t ols-indexer . + + + +## Building OLS manually + +To build OLS you will need to use Java 8 and Maven 3.x. + +To build OLS, in the root directory of OLS, run: +`mvn clean package`. Currently this will fail with the following error: + +`[ERROR] Failed to execute goal on project ols-neo4j: Could not resolve dependencies for project uk.ac.ebi.spot:ols-neo4j:jar:3.2.1-SNAPSHOT: Failed to collect dependencies at org.springframework.data:spring-data-neo4j:jar:3.4.5.RELEASE -> org.neo4j:neo4j-cypher-dsl:jar:2.0.1: Failed to read artifact descriptor for org.neo4j:neo4j-cypher-dsl:jar:2.0.1: Could not transfer artifact org.neo4j:neo4j-cypher-dsl:pom:2.0.1 from/to maven-neo4j (https://m2.neo4j.org/content/repositories/releases/): Failed to transfer file https://m2.neo4j.org/content/repositories/releases/org/neo4j/neo4j-cypher-dsl/2.0.1/neo4j-cypher-dsl-2.0.1.pom with status code 502 -> [Help 1]` + +To correct this, copy the contents of the `build-fix` directory into your Maven +repository under `~/.m2/repository`. + +Run `mvn clean package` again. OLS should now build successfully. + +### Other build errors +Other build errors you may come across are the following: + +1. Wrong version of Java used: + +`[ERROR] Failed to execute goal org.apache.maven.plugins:maven-compiler-plugin:3.1:compile (default-compile) on project ols-solr: Compilation failure: Compilation failure: + [ERROR] /Users/james/OLS/ols-solr/src/main/java/uk/ac/ebi/spot/ols/config/SolrContext.java:[15,24] package javax.annotation does not exist + [ERROR] /Users/james/OLS/ols-solr/src/main/java/uk/ac/ebi/spot/ols/config/SolrContext.java:[25,4] cannot find symbol + [ERROR] symbol: class Resource + [ERROR] location: class uk.ac.ebi.spot.ols.config.SolrContext + [ERROR] -> [Help 1]` -All of the apps are available under the ols-apps module. +This is the error you get when you compile OLS with Java 11. The fix for this +build error is to ensure your Maven installation is indeed using Java 8 for +compilation. - * [ols-apps/ols-solr-app](ols-apps/ols-solr-app) - Spring boot application for building a SOLR index for one of more ontologies. Requires access to a SOLR server. - * [ols-apps/ols-neo4j-app](ols-apps/ols-neo4j-app) - Spring boot application for building a Neo4j index for one of more ontologies. Builds an embedded neo4j database. You can run a Neo4j server that uses the generated neo4j database. +## Customisation -To run a complete local OLS installation you will need a mongodb database. This is a lightweight database that used to store all the ontology configuration and application state information. See here for more information http://www.ebi.ac.uk/ols/docs/installation-guide +It is possible to customise several branding options in `ols-web/src/main/resources/application.properties`: - * [ols-apps/ols-config-importer](ols-apps/ols-config-importer) - Spring boot application for loading config files into the mongodb database. This includes support for reading config files specified using the OBO foundry YAML format. - * [ols-apps/ols-loading-app](ols-apps/ols-loading-app) - Spring boot application for that build the complete OLS indexes. This app fetches ontologies specified in the config files, checks if they have changed form a previous download, and if hey have changed it will create all the necessary SOLR and Neo4j indexes. +* `ols.customisation.debrand` — If set to true, removes the EBI header and footer, documentation, and about page +* `ols.customisation.title` — A custom title for your instance, e.g. "My OLS Instance" +* `ols.customisation.short-title` — A shorter version of the custom title, e.g. "MYOLS" +* `ols.customisation.org` — The organisation hosting your instance - * [ols-web](ols-web) - This will contain the WAR file that can be deployed in tomcat to launch the OLS website and REST API. diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 6559bd68..c04c9c63 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,6 @@ +See https://github.com/EBISPOT/ols/releases for latest releases and notes. + + -------------------------------- OLS 3.1.0 release notes -------------------------------- diff --git a/build-fix/org/neo4j/neo4j-cypher-dsl/2.0.1/_remote.repositories b/build-fix/org/neo4j/neo4j-cypher-dsl/2.0.1/_remote.repositories new file mode 100644 index 00000000..6221beac --- /dev/null +++ b/build-fix/org/neo4j/neo4j-cypher-dsl/2.0.1/_remote.repositories @@ -0,0 +1,4 @@ +#NOTE: This is a Maven Resolver internal implementation file, its format can be changed without prior notice. +#Fri Jan 17 16:17:27 GMT 2020 +neo4j-cypher-dsl-2.0.1.pom>maven-neo4j= +neo4j-cypher-dsl-2.0.1.jar>maven-neo4j= diff --git a/build-fix/org/neo4j/neo4j-cypher-dsl/2.0.1/neo4j-cypher-dsl-2.0.1.jar b/build-fix/org/neo4j/neo4j-cypher-dsl/2.0.1/neo4j-cypher-dsl-2.0.1.jar new file mode 100644 index 00000000..fbdd364b Binary files /dev/null and b/build-fix/org/neo4j/neo4j-cypher-dsl/2.0.1/neo4j-cypher-dsl-2.0.1.jar differ diff --git a/build-fix/org/neo4j/neo4j-cypher-dsl/2.0.1/neo4j-cypher-dsl-2.0.1.jar.sha1 b/build-fix/org/neo4j/neo4j-cypher-dsl/2.0.1/neo4j-cypher-dsl-2.0.1.jar.sha1 new file mode 100644 index 00000000..9873bc00 --- /dev/null +++ b/build-fix/org/neo4j/neo4j-cypher-dsl/2.0.1/neo4j-cypher-dsl-2.0.1.jar.sha1 @@ -0,0 +1 @@ +519d317979935773646356ee2f7923090f7d53e8 \ No newline at end of file diff --git a/build-fix/org/neo4j/neo4j-cypher-dsl/2.0.1/neo4j-cypher-dsl-2.0.1.pom b/build-fix/org/neo4j/neo4j-cypher-dsl/2.0.1/neo4j-cypher-dsl-2.0.1.pom new file mode 100644 index 00000000..ebf5b898 --- /dev/null +++ b/build-fix/org/neo4j/neo4j-cypher-dsl/2.0.1/neo4j-cypher-dsl-2.0.1.pom @@ -0,0 +1,267 @@ + + 4.0.0 + org.neo4j + neo4j-cypher-dsl + 2.0.1 + Neo4j Cypher DSL + Neo4j Cypher DSL + http://components.neo4j.org/${project.artifactId}/${project.version} + + + 2.0.1 + 3.2.4 + org.neo4j.cypherdsl + cypher-dsl + GPL-3-header.txt + + + jar + + + scm:git:git://github.com/neo4j/cypher-dsl.git + scm:git:git@github.com:neo4j/cypher-dsl.git + https://github.com/neo4j/cypher-dsl + neo4j-cypher-dsl-2.0.1 + + + + + GNU General Public License, Version 3 + http://www.gnu.org/licenses/gpl-3.0-standalone.html + The software ("Software") developed and owned by Network Engine for + Objects in Lund AB (referred to in this notice as "Neo Technology") is + licensed under the GNU GENERAL PUBLIC LICENSE Version 3 to all third + parties and that license is included below. + + However, if you have executed an End User Software License and Services + Agreement or an OEM Software License and Support Services Agreement, or + another commercial license agreement with Neo Technology or one of its + affiliates (each, a "Commercial Agreement"), the terms of the license in + such Commercial Agreement will supersede the GNU GENERAL PUBLIC LICENSE + Version 3 and you may use the Software solely pursuant to the terms of + the relevant Commercial Agreement. + + + + + + + com.mysema.querydsl + querydsl-core + ${querydsl.version} + true + + + com.mysema.querydsl + querydsl-lucene3 + ${querydsl.version} + true + + + org.apache.lucene + lucene-core + + + + + com.mysema.querydsl + querydsl-apt + ${querydsl.version} + provided + + + + org.neo4j + neo4j-kernel + ${neo4j.version} + jar + true + + + + org.neo4j + neo4j-lucene-index + ${neo4j.version} + jar + true + + + + org.neo4j + neo4j-cypher + ${neo4j.version} + true + + + + org.codehaus.jackson + jackson-core-asl + 1.9.2 + true + + + + org.codehaus.jackson + jackson-mapper-asl + 1.9.2 + true + + + + org.neo4j + neo4j-kernel + ${neo4j.version} + test-jar + test + + + + junit + junit + 4.11 + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.7 + 1.7 + + + + org.apache.maven.plugins + maven-gpg-plugin + + true + + + + maven-surefire-plugin + + -Xmx300m + + + + com.mysema.maven + maven-apt-plugin + 1.0.2 + + com.mysema.query.apt.QuerydslAnnotationProcessor + + + + test-sources + generate-test-sources + + test-process + + + target/generated-test-sources/querydsl + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + generate-test-sources + + add-source + + + + target/generated-test-sources/querydsl + + + + + + + + maven-assembly-plugin + + + docs-assembly + package + + true + true + + src/main/assemblies/docs-assembly.xml + + + + single + + + + + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + com.mysema.maven + maven-apt-plugin + [1.0.2,) + + test-process + + + + + + + + + + + + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + *.dummy + + + + + + + + releases@repo.neo4j.org + releases@repo.neo4j.org + false + http://m2.neo4j.org/content/repositories/releases + + + snapshots@repo.neo4j.org + snapshots@repo.neo4j.org + false + http://m2.neo4j.org/content/repositories/snapshots + + + + + diff --git a/build-fix/org/neo4j/neo4j-cypher-dsl/2.0.1/neo4j-cypher-dsl-2.0.1.pom.sha1 b/build-fix/org/neo4j/neo4j-cypher-dsl/2.0.1/neo4j-cypher-dsl-2.0.1.pom.sha1 new file mode 100644 index 00000000..ceca2da9 --- /dev/null +++ b/build-fix/org/neo4j/neo4j-cypher-dsl/2.0.1/neo4j-cypher-dsl-2.0.1.pom.sha1 @@ -0,0 +1 @@ +5542a7a246e3843fc05941014bd803e34095a35a \ No newline at end of file diff --git a/config/obo-config.yaml b/config/obo-config.yaml new file mode 100644 index 00000000..a8d1123c --- /dev/null +++ b/config/obo-config.yaml @@ -0,0 +1,34 @@ +name: OBO Foundry +title: The OBO Foundry +markdown: kramdown +highlighter: rouge +baseurl: / +imgurl: /images +repo: https://github.com/OBOFoundry/OBOFoundry.github.io/ +repo_src: https://github.com/OBOFoundry/OBOFoundry.github.io/blob/master/ +author: + name: OBO Technical WG +ontologies: +- activity_status: active + build: {checkout: 'git clone https://github.com/EBISPOT/duo.git', path: ., system: git} + contact: {email: mcourtot@gmail.com, github: mcourtot, label: Melanie Courtot} + dependencies: + - {id: iao} + - {id: bfo} + description: DUO is an ontology which represent data use conditions. + homepage: https://github.com/EBISPOT/DUO + id: duo + jobs: + - {id: 'https://travis-ci.org/EBISPOT/DUO', type: travis-ci} + layout: ontology_detail + license: {label: CC-BY, logo: 'http://mirrors.creativecommons.org/presskit/buttons/80x15/png/by.png', + url: 'http://creativecommons.org/licenses/by/3.0/'} + ontology_purl: http://purl.obolibrary.org/obo/duo.owl + products: + - {id: duo.owl, ontology_purl: 'http://purl.obolibrary.org/obo/duo.owl'} + title: The Data Use Ontology + tracker: https://github.com/EBISPOT/DUO/issues + preferred_root_term: + - http://purl.obolibrary.org/obo/DUO_0000001 + - http://purl.obolibrary.org/obo/DUO_0000017 + - http://purl.obolibrary.org/obo/OBI_0000066 \ No newline at end of file diff --git a/config/ols-config.yaml b/config/ols-config.yaml new file mode 100644 index 00000000..2c161a5f --- /dev/null +++ b/config/ols-config.yaml @@ -0,0 +1,4 @@ +"@context": + +ontologies: + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..ce84ae49 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,46 @@ +version: '2' +services: + solr: + image: ebispot/ols-solr:latest + environment: + - SOLR_HOME=/mnt/solr-config + ports: + - 8983:8983 + volumes: + - ols-solr-data:/var/solr + - ./ols-solr/src/main/solr-5-config:/mnt/solr-config + command: ["-Dsolr.solr.home=/mnt/solr-config", "-Dsolr.data.dir=/var/solr", "-f"] + mongo: + image: mongo:3.2.9 + ports: + - 27017:27017 + volumes: + - ols-mongo-data:/data/db + command: + - mongod + ols-web: + build: + context: . + dockerfile: ols-web/Dockerfile + depends_on: + - solr + - mongo + links: + - solr + - mongo + environment: + - spring.data.solr.host=http://solr:8983/solr + - spring.data.mongodb.host=mongo + - OLS_HOME=/mnt/ + volumes: + - ols-neo4j-data:/mnt/neo4j + ports: + - 8080:8080 +volumes: + ols-solr-data: + external: true + ols-mongo-data: + external: true + ols-neo4j-data: + external: true + diff --git a/documentation/uptake/New OLS ontology request.xlsx b/documentation/uptake/New OLS ontology request.xlsx new file mode 100644 index 00000000..c3af7fcd Binary files /dev/null and b/documentation/uptake/New OLS ontology request.xlsx differ diff --git a/ols-apps/ols-config-importer/Dockerfile b/ols-apps/ols-config-importer/Dockerfile new file mode 100644 index 00000000..d2686c55 --- /dev/null +++ b/ols-apps/ols-config-importer/Dockerfile @@ -0,0 +1,12 @@ + +FROM maven:3.6-jdk-8 AS build +RUN mkdir /opt/ols +COPY . /opt/ols/ +COPY build-fix/. /root/.m2/repository/ +RUN cd /opt/ols && ls && mvn clean package -DskipTests + +FROM openjdk:8-jre-alpine +RUN apk add bash +COPY --from=build /opt/ols/ols-apps/ols-config-importer/target/ols-config-importer.jar /opt/ols-config-importer.jar +EXPOSE 8080 +ENTRYPOINT ["java", "-jar", "/opt/ols-config-importer.jar", "--ols.ontology.config=file:///config/ols-config.yaml","--ols.obofoundry.ontology.config=file:///config/obo-config.yaml"] diff --git a/ols-apps/ols-config-importer/pom.xml b/ols-apps/ols-config-importer/pom.xml index cd1d68a4..be34c832 100644 --- a/ols-apps/ols-config-importer/pom.xml +++ b/ols-apps/ols-config-importer/pom.xml @@ -1,11 +1,9 @@ - + uk.ac.ebi.spot ols-parent - 3.1.1 + 3.2.1-SNAPSHOT ../../ols-parent/pom.xml @@ -33,7 +31,6 @@ org.yaml snakeyaml - 1.15 diff --git a/ols-apps/ols-config-importer/src/main/java/uk/ac/ebi/spot/ols/DocumentUpdater.java b/ols-apps/ols-config-importer/src/main/java/uk/ac/ebi/spot/ols/DocumentUpdater.java index e63a36f6..7a09f85a 100644 --- a/ols-apps/ols-config-importer/src/main/java/uk/ac/ebi/spot/ols/DocumentUpdater.java +++ b/ols-apps/ols-config-importer/src/main/java/uk/ac/ebi/spot/ols/DocumentUpdater.java @@ -21,17 +21,19 @@ public class DocumentUpdater { /** - * This method updates fileds in an ontology document with info from a new ontology document + * This method updates fields in an ontology document with info from a new ontology document * @param originalDocument the original document that needs updating - * @param newDocument the new document that conatins the data that needs to be added to the original document + * @param newDocument the new document that contains the data that needs to be added to the original document * @return the original document iwth fields updated */ public static OntologyDocument updateFields(OntologyDocument originalDocument, OntologyResourceConfig newDocument) { // if location has changed, update the info - if (newDocument.getFileLocation() != null && !originalDocument.getConfig().getFileLocation().equals(newDocument.getFileLocation())) { - log.info("Location of " + newDocument.getNamespace() + " changed from " + originalDocument.getConfig().getFileLocation() + " to " + newDocument.getFileLocation()); + if (newDocument.getFileLocation() != null && + !originalDocument.getConfig().getFileLocation().equals(newDocument.getFileLocation())) { + log.info("Location of " + newDocument.getNamespace() + " changed from " + + originalDocument.getConfig().getFileLocation() + " to " + newDocument.getFileLocation()); originalDocument.getConfig().setFileLocation(newDocument.getFileLocation()); originalDocument.setStatus(Status.TOLOAD); } @@ -48,17 +50,23 @@ public static OntologyDocument updateFields(OntologyDocument originalDocument, O } // check description - if (newDocument.getDescription() != null && !newDocument.getDescription().equals(originalDocument.getConfig().getDescription()) && !dontUpdate.contains(OntologyDefaults.DEFINITION)) { + if (newDocument.getDescription() != null && + !newDocument.getDescription().equals(originalDocument.getConfig().getDescription()) && + !dontUpdate.contains(OntologyDefaults.DEFINITION)) { originalDocument.getConfig().setDescription(newDocument.getDescription()); } // check homepage - if (newDocument.getHomepage() != null && !newDocument.getHomepage().equals(originalDocument.getConfig().getHomepage()) && !dontUpdate.contains(OntologyDefaults.HOMEPAGE)) { + if (newDocument.getHomepage() != null && + !newDocument.getHomepage().equals(originalDocument.getConfig().getHomepage()) && + !dontUpdate.contains(OntologyDefaults.HOMEPAGE)) { originalDocument.getConfig().setHomepage(newDocument.getHomepage()); } // check mailing list - if (newDocument.getMailingList() != null && !newDocument.getMailingList().equals(originalDocument.getConfig().getMailingList()) && !dontUpdate.contains(OntologyDefaults.MAILINGLIST)) { + if (newDocument.getMailingList() != null && + !newDocument.getMailingList().equals(originalDocument.getConfig().getMailingList()) && + !dontUpdate.contains(OntologyDefaults.MAILINGLIST)) { originalDocument.getConfig().setMailingList(newDocument.getMailingList()); } @@ -76,13 +84,17 @@ public static OntologyDocument updateFields(OntologyDocument originalDocument, O originalDocument.getConfig().setLabelProperty(newDocument.getLabelProperty()); originalDocument.getConfig().setDefinitionProperties(newDocument.getDefinitionProperties()); originalDocument.getConfig().setSynonymProperties(newDocument.getSynonymProperties()); - originalDocument.getConfig().setHierarchicalProperties(newDocument.getHierarchicalProperties()); + originalDocument.getConfig().setPreferredRootTerms(newDocument.getPreferredRootTerms()); if (!newDocument.getBaseUris().isEmpty()) { originalDocument.getConfig().setBaseUris(newDocument.getBaseUris()); } + originalDocument.getConfig().setAllowDownload(newDocument.getAllowDownload()); + + + // Henriette To do: Remove because this is not used. if (newDocument.getHiddenProperties() != null) { originalDocument.getConfig().setHiddenProperties(newDocument.getHiddenProperties()); } diff --git a/ols-apps/ols-config-importer/src/main/java/uk/ac/ebi/spot/ols/YamlLoader.java b/ols-apps/ols-config-importer/src/main/java/uk/ac/ebi/spot/ols/YamlLoader.java index ca0257fd..87cfe9d8 100644 --- a/ols-apps/ols-config-importer/src/main/java/uk/ac/ebi/spot/ols/YamlLoader.java +++ b/ols-apps/ols-config-importer/src/main/java/uk/ac/ebi/spot/ols/YamlLoader.java @@ -31,7 +31,6 @@ @SpringBootApplication -@EnableAutoConfiguration @Configuration public class YamlLoader implements CommandLineRunner { diff --git a/ols-apps/ols-config-importer/src/main/resources/application.properties b/ols-apps/ols-config-importer/src/main/resources/application.properties index e4448041..144826b6 100644 --- a/ols-apps/ols-config-importer/src/main/resources/application.properties +++ b/ols-apps/ols-config-importer/src/main/resources/application.properties @@ -3,6 +3,4 @@ logging.level.ch.qos.logback.*=ERROR spring.main.show_banner=false spring.data.mongodb.database ols -ols.obofoundry.ontology.config https://raw.githubusercontent.com/OBOFoundry/OBOFoundry.github.io/master/_config.yml -#ols.ontology.config http://www.cropontology.org/metadata -#ols.obofoundry.dontclassify ncbitaxon,gaz +ols.obofoundry.ontology.config obo-config.yaml diff --git a/ols-apps/ols-loading-app/src/main/resources/logback.xml b/ols-apps/ols-config-importer/src/main/resources/logback-spring.xml similarity index 50% rename from ols-apps/ols-loading-app/src/main/resources/logback.xml rename to ols-apps/ols-config-importer/src/main/resources/logback-spring.xml index e17a6de5..2354ac1d 100644 --- a/ols-apps/ols-loading-app/src/main/resources/logback.xml +++ b/ols-apps/ols-config-importer/src/main/resources/logback-spring.xml @@ -1,29 +1,27 @@ - - %d{ISO8601} %p %t %c{0}.%M - %m%n utf8 - - - + + ols-config-importer.log + false + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{35} %M - %msg%n + + - - - - + - + \ No newline at end of file diff --git a/ols-apps/ols-config-importer/src/main/resources/logback.xml b/ols-apps/ols-config-importer/src/main/resources/logback.xml deleted file mode 100644 index 40308035..00000000 --- a/ols-apps/ols-config-importer/src/main/resources/logback.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - %d{ISO8601} %p %t %c{0}.%M - %m%n - utf8 - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/ols-apps/ols-config-importer/src/main/resources/obo-config.yaml b/ols-apps/ols-config-importer/src/main/resources/obo-config.yaml index 6f0ba79e..fdde3ea1 100644 --- a/ols-apps/ols-config-importer/src/main/resources/obo-config.yaml +++ b/ols-apps/ols-config-importer/src/main/resources/obo-config.yaml @@ -1,478 +1,550 @@ -"@context": - "@base": http://purl.obolibrary.org/obo/ - rdf: http://www.w3.org/1999/02/22-rdf-syntax-ns# - rdfs: http://www.w3.org/2000/01/rdf-schema# - owl: http://www.w3.org/2002/07/owl# - xsd: http://www.w3.org/2001/XMLSchema# - dc: http://purl.org/dc/terms/ - dce: http://purl.org/dc/elements/1.1/ - dcterms: http://purl.org/dc/terms/1.1/ - faldo: http://biohackathon.org/resource/faldo# - foaf: http://xmlns.com/foaf/0.1/ - oa: http://www.w3.org/ns/oa# - idot: http://identifiers.org/ - void: http://rdfs.org/ns/void# - doap: http://usefulinc.com/ns/doap# - pav: http://purl.org/pav/ - - id: "@id" - label: rdfs:label - comment: rdfs:comment - description: dce:description - source: dce:source - publisher: dce:publisher - mailing_list: doap:mailing-list - createdWith: pav:createdWith - wasDerivedFrom: pav:wasDerivedFrom - preferredPrefix: idot:preferredPrefix - alternatePrefix: idot:alternatePrefix - identifierPattern: idot:identifierPattern - exampleIdentifier: idot:exampleIdentifier - - Class: owl:Class - - products: void:subset - - type: - "@id": rdf:type - "@type": "@id" - created: - "@id": dcterms:created - "@type": xsd:dateTime - creator: - "@id": dcterms:creator - "@type": "@id" - depiction: - "@id": foaf:depiction - "@type": "@id" - publications: dcterms:publication - source: dce:source - subClassOf: owl:subClassOf - title: dce:title - homepage: - "@id": foaf:homepage - "@type": "@id" - page: - "@id": foaf:page - "@type": "@id" - ontologies: dcterms:hasPart - - -## ---------------------------------------- -## BUSINESS STARTS HERE -## ---------------------------------------- - -id: http://obolibrary.org -title: OBO Library VERY EXPERIMENTAL BROWSER -description: "A set of ontology libraries for doing useful stuff. This is a demo, DO NOT TRUST ANY INFO YOU SEE HERE" - +name: OBO Foundry +title: The OBO Foundry +markdown: kramdown +highlighter: rouge +baseurl: / +imgurl: /images +repo: https://github.com/OBOFoundry/OBOFoundry.github.io/ +repo_src: https://github.com/OBOFoundry/OBOFoundry.github.io/blob/master/ +author: + name: OBO Technical WG ontologies: - -## UBERON - - - id: uberon - type: owl:Ontology - label: Uberon - title: Uberon multi-species anatomy ontology - description: Uberon is an integrated cross-species anatomy ontology representing a variety of entities classified according to traditional anatomical criteria such as structure, function and developmental lineage. The ontology includes comprehensive relationships to taxon-specific anatomical ontologies, allowing integration of functional, phenotype and expression data - homepage: http://uberon.org - page: http://en.wikipedia.org/wiki/Uberon - twitter: uberanat - google_plus: "+UberonOrg" - mailing_list: https://lists.sourceforge.net/lists/listinfo/obo-anatomy + - activity_status: active + build: {checkout: 'git clone https://github.com/EBISPOT/duo.git', path: ., system: git} + contact: {email: mcourtot@gmail.com, github: mcourtot, label: Melanie Courtot} + dependencies: + - {id: iao} + - {id: bfo} + description: DUO is an ontology which represent data use conditions. + homepage: https://github.com/EBISPOT/DUO + id: duo + jobs: + - {id: 'https://travis-ci.org/EBISPOT/DUO', type: travis-ci} + layout: ontology_detail + license: {label: CC-BY, logo: 'http://mirrors.creativecommons.org/presskit/buttons/80x15/png/by.png', + url: 'http://creativecommons.org/licenses/by/3.0/'} + ontology_purl: http://purl.obolibrary.org/obo/duo.owl + products: + - {id: duo.owl, ontology_purl: 'http://purl.obolibrary.org/obo/duo.owl'} + title: The Data Use Ontology + tracker: https://github.com/EBISPOT/DUO/issues + preferred_root_term: + - http://purl.obolibrary.org/obo/DUO_0000001 + - http://purl.obolibrary.org/obo/DUO_0000017 + - http://purl.obolibrary.org/obo/OBI_0000066 + - activity_status: active + browsers: + - {label: Monarch, title: Monarch Initiative Disease Browser, url: 'https://monarchinitiative.org/disease/MONDO:0019609'} + canonical: mondo.owl + contact: {email: vasilevs@ohsu.edu, github: nicolevasilevsky, label: Nicole Vasilevsky} + description: An ontology that harmonizes multiple disease resources. + domain: disease + homepage: https://monarch-initiative.github.io/mondo + id: mondo + label: Mondo + layout: ontology_detail + license: {label: CC BY 3.0, logo: 'http://mirrors.creativecommons.org/presskit/buttons/80x15/png/by.png', + url: 'http://creativecommons.org/licenses/by/3.0/'} + mailing_list: https://groups.google.com/group/mondo-users + ontology_purl: http://purl.obolibrary.org/obo/mondo.owl + products: + - {description: 'Complete ontology, plus inter-ontology equivalence axioms. Uses + MONDO IDs.', format: owl-rdf/xml, id: mondo.owl, is_canonical: true, ontology_purl: 'http://purl.obolibrary.org/obo/mondo.owl', + title: Main OWL edition} + - {derived_from: mondo.owl, description: 'As OWL, but omits equivalence axioms. + xrefs can be used as proxy for equivalence. Uses Mondo IDs', format: obo, id: mondo.obo, + ontology_purl: 'http://purl.obolibrary.org/obo/mondo.obo', title: obo-format edition} + - {derived_from: mondo.owl, description: Equivalent to the OWL edition, format: obo, + id: mondo.json, ontology_purl: 'http://purl.obolibrary.org/obo/mondo.json', title: json + edition} + taxon: {id: 'NCBITaxon:33208', label: Metazoa} + title: Mondo Disease Ontology + tracker: https://github.com/monarch-initiative/mondo-build/issues + usages: + - description: Mondo is used by the Monarch Initiative for disease annotations. + examples: + - {url: 'https://monarchinitiative.org/phenotype/HP:0001300#diseases'} + reference: https://academic.oup.com/nar/article/45/D1/D712/2605791 + type: annotation + user: https://monarchinitiative.org/ + - activity_status: active + biosharing: https://www.biosharing.org/bsg-000016 + browsers: + - label: RGD + title: Gene Ontology AmiGO 2 Browser + url: http://rgd.mcw.edu/rgdweb/ontology/view.html?acc_id=UBERON:0001062 + - label: AmiGO (SUBSET) + title: Gene Ontology AmiGO 2 Browser + url: http://amigo.geneontology.org/amigo/term/UBERON:0001062#display-lineage-tab + - label: Bgee (gene expression) + title: Bgee gene expression queries + url: http://bgee.org/?page=gene + - label: FANTOM5 + title: FANTOM5 Data Portal + url: http://fantom.gsc.riken.jp/5/sstar/UBERON:0001890 + - label: KnowledgeSpace + title: INCF KnowledgeSpace Portal + url: https://knowledge-space.org/index.php/pages/view/UBERON:0000061 + build: + checkout: svn --ignore-externals co http://svn.code.sf.net/p/obo/svn/uberon/trunk + email_cc: cjmungall@lbl.gov + infallible: 1 + method: vcs + system: svn canonical: uberon.owl - taxon: - id: NCBITaxon:33208 - label: Metazoa - domain: anatomy - repository: https://github.com/obophenotype/uberon - tracker: https://github.com/obophenotype/uberon/issues - releases: http://purl.obolibrary.org/obo/uberon/releases/ contact: email: cjmungall@lbl.gov label: Chris Mungall + dependencies: + - id: go + subset: uberon/go_import.owl + - id: cl + subset: uberon/cl_import.owl + - id: chebi + subset: uberon/chebi_import.owl + - id: pr + subset: uberon/pr_import.owl + depicted_by: http://uberon.github.io/images/u-logo.jpg + description: An integrated cross-species anatomy ontology covering animals and bridging + multiple species-specific ontologies + domain: anatomy + exampleClass: UBERON_0002046 + funded_by: + - NIH R24OD011883 + - NIH R01HG004838 + - NIH P41HG002273 + - NSF DEB-0956049 + google_plus: +UberonOrg + homepage: http://uberon.org + id: uberon + label: Uberon + layout: ontology_detail + license: + label: CC-BY + logo: http://mirrors.creativecommons.org/presskit/buttons/80x15/png/by.png + url: http://creativecommons.org/licenses/by/3.0/ + mailing_list: https://lists.sourceforge.net/lists/listinfo/obo-anatomy + ontology_purl: http://purl.obolibrary.org/obo/uberon.owl + page: http://en.wikipedia.org/wiki/Uberon + products: + - description: core ontology + id: uberon.owl + is_canonical: true + ontology_purl: http://purl.obolibrary.org/obo/uberon.owl + title: Uberon + type: owl:Ontology + - description: Axioms defined within Uberon and to be used in imports for other + ontologies + id: uberon/uberon-base.owl + ontology_purl: http://purl.obolibrary.org/obo/uberon/uberon-base.owl + page: https://github.com/INCATools/ontology-development-kit/issues/50 + title: Uberon base ontology + - description: Uberon extended + id: uberon/ext.owl + mireots_from: ncbitaxon + ontology_purl: http://purl.obolibrary.org/obo/uberon/ext.owl + title: Uberon edition that includes subsets of other ontologies and axioms connecting + to them + type: owl:Ontology + - description: Uberon edition that excludes external ontologies and most relations + format: obo + id: uberon/basic.obo + ontology_purl: http://purl.obolibrary.org/obo/uberon/basic.obo + title: Uberon basic + type: obo-basic-ontology + - connects: + - id: uberon + - id: zfa + description: Taxonomic equivalence axioms connecting zebrafish-specific classes + to generic uberon counterparts + id: uberon/bridge/uberon-bridge-to-zfa.owl + ontology_purl: http://purl.obolibrary.org/obo/uberon/bridge/uberon-bridge-to-zfa.owl + page: https://github.com/obophenotype/uberon/wiki/inter-anatomy-ontology-bridge-ontologies + title: Uberon bridge to ZFA + type: BridgeOntology + - connects: + - id: uberon + - id: ma + description: Taxonomic equivalence axioms connecting adult mouse specific classes + to generic uberon counterparts + id: uberon/bridge/uberon-bridge-to-ma.owl + ontology_purl: http://purl.obolibrary.org/obo/uberon/bridge/uberon-bridge-to-ma.owl + page: https://github.com/obophenotype/uberon/wiki/inter-anatomy-ontology-bridge-ontologies + title: Uberon bridge to MA + type: BridgeOntology + - description: Extended uberon plus all metazoan ontologies + id: uberon/composite-metazoan.owl + mireots_from: ehdaa2 + ontology_purl: http://purl.obolibrary.org/obo/uberon/composite-metazoan.owl + page: https://github.com/obophenotype/uberon/wiki/Multi-species-composite-ontologies + taxon: Metazoa + title: Uberon composite metazoan ontology + type: MergedOntology + - id: uberon/composite-vertebrate.owl + mireots_from: ehdaa2 + ontology_purl: http://purl.obolibrary.org/obo/uberon/composite-vertebrate.owl + page: https://github.com/obophenotype/uberon/wiki/Multi-species-composite-ontologies + taxon: Metazoa + title: Uberon composite vertebrate ontology + type: MergedOntology publications: - id: http://www.ncbi.nlm.nih.gov/pubmed/22293552 - depicted_by: http://uberon.github.io/images/u-logo.jpg + title: Uberon, an integrative multi-species anatomy ontology + - id: http://www.ncbi.nlm.nih.gov/pubmed/25009735 + title: Unification of multi-species vertebrate anatomy ontologies for comparative + biology in Uberon redirects: - - match: "releases/" - url: "http://svn.code.sf.net/p/obo/svn/uberon/releases/" - - match: "" - url: "http://berkeleybop.org/ontologies/uberon/" - dependencies: - - id: go - subset: uberon/go_import.owl - - id: cl - subset: uberon/cl_import.owl - - id: chebi - subset: uberon/chebi_import.owl - - id: pr - subset: uberon/pr_import.owl - products: - - id: uberon.owl - type: owl:Ontology - title: Uberon - is_canonical: true - - id: uberon/ext.owl - type: owl:Ontology - description: Uberon extended - title: Uberon edition that includes subsets of other ontologies - mireots_from: cl - mireots_from: pr - mireots_from: envo - mireots_from: go - mireots_from: pato - mireots_from: ncbitaxon - - id: uberon/basic.obo - title: Uberon basic - description: Uberon edition that excludes external ontologies and most relations - format: obo - type: obo-basic-ontology - - id: uberon/bridge/uberon-bridge-to-zfa.owl - type: BridgeOntology - page: https://github.com/obophenotype/uberon/wiki/inter-anatomy-ontology-bridge-ontologies - title: Uberon bridge to ZFA - description: Taxonomic equivalence axioms connecting zebrafish-specific classes to generic uberon counterparts - connects: - - id: uberon - - id: zfa - - id: uberon/bridge/uberon-bridge-to-ma.owl - type: BridgeOntology - page: https://github.com/obophenotype/uberon/wiki/inter-anatomy-ontology-bridge-ontologies - title: Uberon bridge to MA - description: Taxonomic equivalence axioms connecting adult mouse specific classes to generic uberon counterparts - connects: - - id: uberon - - id: ma - - id: uberon/composite-metazoan.owl - type: MergedOntology - title: Uberon composite metazoan ontology - page: https://github.com/obophenotype/uberon/wiki/Multi-species-composite-ontologies - taxon: Metazoa - mireots_from: zfa - mireots_from: xao - mireots_from: fbbt - mireots_from: wbbt - mireots_from: ma - mireots_from: fma - mireots_from: emapa - mireots_from: ehdaa2 - - id: uberon/composite-vertebrate.owl - type: MergedOntology - title: Uberon composite vertebrate ontology - page: https://github.com/obophenotype/uberon/wiki/Multi-species-composite-ontologies - taxon: Metazoa - mireots_from: zfa - mireots_from: xao - mireots_from: fbbt - mireots_from: wbbt - mireots_from: ma - mireots_from: fma - mireots_from: emapa - mireots_from: ehdaa2 - -## GO - - - id: go - label: GO - title: Gene Ontology - twitter: news4go - tracker: http://sourceforge.net/p/geneontology/ontology-requests/ - termgenie: http://go.termgenie.org - taxon: - id: NCBITaxon:1 - label: All life - domain: biology - dependencies: - - id: uberon - subset: go/extensions/uberon_import.owl - - id: cl - subset: go/extensions/cl_import.owl - - id: ncbitaxon - subset: go/extensions/ncbitaxon_import.owl - - id: ro - subset: go/extensions/ro_import.owl - - id: go/extensions/go-bridge-to-nifstd.owl - type: BridgeOntology - title: GO bridge to NIFSTD - description: Bridging axioms between nifstd and go - connects: - - id: nifstd - - id: go - products: - - id: go.owl - depicted_by: http://geneontology.org/sites/default/files/go-logo-icon.mini__0.png - -## CL - - - id: cl - label: Cell Ontology - title: Cell Ontology - description: The Cell Ontology is designed as a structured controlled vocabulary for cell types. This ontology was constructed for use by the model organism and other bioinformatics databases, where there is a need for a controlled vocabulary of cell types. This ontology is not organism specific it covers cell types from prokaryotes to mammals. However, it excludes plant cell types, which are covered by PO. + - match: releases/ + url: http://svn.code.sf.net/p/obo/svn/uberon/releases/ + - match: '' + url: http://berkeleybop.org/ontologies/uberon/ + releases: http://purl.obolibrary.org/obo/uberon/releases/ + repository: https://github.com/obophenotype/uberon taxon: id: NCBITaxon:33208 label: Metazoa - domain: cells - tracker: https://code.google.com/p/cell-ontology/issues/list - termgenie: http://cl.termgenie.org - mailing_list: https://lists.sourceforge.net/lists/listinfo/obo-cell-type + title: Uberon multi-species anatomy ontology + tracker: https://github.com/obophenotype/uberon/issues + twitter: uberanat + type: owl:Ontology + usages: + - description: Bgee is a database to retrieve and compare gene expression patterns + between animal species. Bgee in using Uberon to annotate the site of expression, + and Bgee curators one the major contributors to the ontology. + examples: + - description: Uberon terms used to annotate expression of human hemoglobin subunit + beta + url: http://bgee.org/?page=gene&gene_id=ENSG00000244734 + seeAlso: https://www.biosharing.org/biodbcore-000228 + type: annotation + user: http://bgee.org/ + - description: The National Human Genome Research Institute (NHGRI) launched a public + research consortium named ENCODE, the Encyclopedia Of DNA Elements, in September + 2003, to carry out a project to identify all functional elements in the human + genome sequence. The ENCODE DCC users Uberon to annotate samples + reference: https://doi.org/10.1093/database/bav010 + seeAlso: https://www.biosharing.org/biodbcore-000034 + type: annotation + user: https://www.encodeproject.org/ + - description: FANTOM5 is using Uberon and CL to annotate samples allowing for transcriptome + analyses with cell-type and tissue-level specificity. + examples: + - description: FANTOM5 samples annotated to telencephalon or its parts + url: http://fantom.gsc.riken.jp/5/sstar/UBERON:0001893 + type: annotation + user: http://fantom5-collaboration.gsc.riken.jp/ + - description: Querying expression and phenotype data + type: query + user: https://monarchinitiative.org/ + - description: Querying for functional annotations relevant to a tissue + examples: + - description: GO annotations relevant to the uberon class for brain + url: http://amigo.geneontology.org/amigo/term/UBERON:0000955 + label: GO Database + type: query + user: https://geneontology.org/ + - description: The Phenoscape project is both a major driver of and contributor + to Uberon, contibuting thousands of terms. The teleost (bony fishes) component + of Uberon was derived from the Teleost Anatomy Ontology, developed by the Phenoscape + group. Most of the high level design of the skeletal system comes from the Vertebrate + Skeletal Anatomy Ontology (VSAO), also created by the Phenoscape group. Phenoscape + curators continue to extend the ontology, covering a wide variety of tetrapod + structures, with an emphasis on the appendicular system. + label: Phenoscape + user: http://phenoscape.org + - label: Neuroscience Information Framework + type: Database + url: https://neuinfo.org/ + - label: SciCrunch + type: Database + url: https://scicrunch.org/ + - label: SCPortalen + reference: https://doi.org/10.1093/nar/gkx949 + type: Database + url: http://single-cell.clst.riken.jp/ + - description: ChEMBL uses Uberon to describe organ/tissue information in assays + label: ChEMBL + reference: https://doi.org/10.1093/nar/gky1075 + type: Database + url: https://www.ebi.ac.uk/chembl/ + wikidata_template: https://en.wikipedia.org/wiki/Template:Uberon + - activity_status: active + browsers: + - label: AmiGO + title: Gene Ontology AmiGO 2 Browser + url: http://amigo.geneontology.org/amigo/term/GO:0008150#display-lineage-tab + contact: + email: suzia@stanford.edu + label: Suzi Aleksander dependencies: - - id: uberon - - id: go - canonical: cl.owl - products: - - id: cl.owl - - id: cl.obo - - id: cl/cl-basic.obo - -## RO - - - id: ro - title: Relations Ontology - canonical: ro.owl - tracker: https://code.google.com/p/obo-relations/issues/list - mailing_list: https://lists.sourceforge.net/lists/listinfo/obo-relations - domain: relations - jobs: - - id: http://build.berkeleybop.org/job/build-ro - type: DryRunBuild + - id: uberon + subset: go/extensions/uberon_import.owl + - id: cl + subset: go/extensions/cl_import.owl + - id: ncbitaxon + subset: go/extensions/ncbitaxon_import.owl + - id: ro + subset: go/extensions/ro_import.owl + - connects: + - id: nifstd + - id: go + description: Bridging axioms between nifstd and go + id: go/extensions/go-bridge-to-nifstd.owl + publications: + - id: http://www.ncbi.nlm.nih.gov/pubmed/24093723 + title: 'The Gene Ontology (GO) Cellular Component Ontology: integration with + SAO (Subcellular Anatomy Ontology) and other recent developments.' + title: GO bridge to NIFSTD + type: BridgeOntology + depicted_by: http://geneontology.org/assets/go-logo-icon.mini.png + description: An ontology for describing the function of genes and gene products + domain: biology + facebook: https://www.facebook.com/Gene-Ontology-305908656519/ + homepage: http://geneontology.org/ + id: go + in_foundry_order: 1 + integration_server: http://build.berkeleybop.org/view/GO + label: GO + layout: ontology_detail + license: + label: CC BY 4.0 + logo: http://mirrors.creativecommons.org/presskit/buttons/80x15/png/by.png + url: https://creativecommons.org/licenses/by/4.0/ + ontology_purl: http://purl.obolibrary.org/obo/go.owl products: - - id: ro.owl - - id: ro.obo - - id: ro/ro-interaction.owl - -## CHEBI - - - id: chebi + - description: The main ontology in OWL. This is self contained and does not have + connections to other OBO ontologies + id: go.owl + ontology_purl: http://purl.obolibrary.org/obo/go.owl + page: http://geneontology.org/page/download-ontology + title: GO (OWL edition) + - description: Equivalent to go.owl, in obo format + id: go.obo + ontology_purl: http://purl.obolibrary.org/obo/go.obo + page: http://geneontology.org/page/download-ontology + title: GO (OBO Format edition) + - description: Equivalent to go.owl, in obograph json format + id: go.json + ontology_purl: http://purl.obolibrary.org/obo/go.json + page: https://github.com/geneontology/obographs/ + title: GO (JSON edition) + - description: The main ontology plus axioms connecting to select external ontologies, + with subsets of those ontologies + id: go/extensions/go-plus.owl + ontology_purl: http://purl.obolibrary.org/obo/go/extensions/go-plus.owl + page: http://geneontology.org/page/download-ontology + title: GO-Plus + - description: The main ontology plus axioms connecting to select external ontologies, + excluding the external ontologies themselves + id: go/go-base.owl + ontology_purl: http://purl.obolibrary.org/obo/go/go-base.owl + page: http://geneontology.org/page/download-ontology + title: GO Base Module + - description: As go-plus.owl, in obographs json format + id: go/extensions/go-plus.json + ontology_purl: http://purl.obolibrary.org/obo/go/extensions/go-plus.json + page: https://github.com/geneontology/obographs/ + title: GO-Plus + - description: The main ontology plus axioms connecting to select external ontologies + id: go/go-basic.obo + ontology_purl: http://purl.obolibrary.org/obo/go/go-basic.obo + page: http://geneontology.org/page/download-ontology + title: GO-Basic, Filtered, for use with legacy tools + - description: As go-basic.obo, in json format + id: go/go-basic.json + ontology_purl: http://purl.obolibrary.org/obo/go/go-basic.json + page: http://geneontology.org/page/download-ontology + title: GO-Basic, Filtered, for use with legacy tools (JSON) + - description: Classes added to ncbitaxon for groupings such as prokaryotes + id: go/extensions/go-taxon-groupings.owl + ontology_purl: http://purl.obolibrary.org/obo/go/extensions/go-taxon-groupings.owl + page: http://geneontology.org/page/download-ontology + title: GO Taxon Groupings + - description: Equivalent to go.owl, but released daily. Note the snapshot release + is not archived. + id: go/snapshot/go.owl + ontology_purl: http://purl.obolibrary.org/obo/go/snapshot/go.owl + page: http://geneontology.org/page/download-ontology + title: GO (OWL edition), daily snapshot release + - description: Equivalent to go.owl, but released daily. Note the snapshot release + is not archived. + id: go/snapshot/go.obo + ontology_purl: http://purl.obolibrary.org/obo/go/snapshot/go.obo + page: http://geneontology.org/page/download-ontology + title: GO (OBO Format edition), daily snapshot release + review: + date: 2010 + taxon: + id: NCBITaxon:1 + label: All life + title: Gene Ontology + tracker: https://github.com/geneontology/go-ontology/issues/ + twitter: news4go + usages: + - description: The GO ontology is used by the GO consortium for functional annotation + of genes + examples: + - description: annotations to transmembrane transport + url: http://amigo.geneontology.org/amigo/term/GO:0055085 + type: annotation + user: http://geneontology.org + - activity_status: active alternatePrefix: ChEBI - title: Chemical entities of biological interest - description: A structured classification of chemical compounds of biological relevance. + browsers: + - label: CHEBI + title: EBI CHEBI Browser + url: http://www.ebi.ac.uk/chebi/chebiOntology.do?treeView=true&chebiId=CHEBI:24431#graphView + build: + infallible: 1 + method: obo2owl + source_url: ftp://ftp.ebi.ac.uk/pub/databases/chebi/ontology/chebi.obo contact: - email: hastings@ebi.ac.uk - label: Janna Hastings + email: amalik@ebi.ac.uk + github: amalik01 + label: Adnan Malik + depicted_by: https://www.ebi.ac.uk/chebi/images/ChEBI_logo.png + description: A structured classification of molecular entities of biological interest + focusing on 'small' chemical compounds. domain: biochemistry homepage: http://www.ebi.ac.uk/chebi + id: chebi + in_foundry_order: 1 + layout: ontology_detail + license: + label: CC-BY 4.0 + logo: http://mirrors.creativecommons.org/presskit/buttons/80x15/png/by.png + url: https://creativecommons.org/licenses/by/4.0/ + ontology_purl: http://purl.obolibrary.org/obo/chebi.owl page: http://www.ebi.ac.uk/chebi/init.do?toolBarForward=userManual products: - id: chebi.owl + ontology_purl: http://purl.obolibrary.org/obo/chebi.owl - id: chebi.obo - tracker: https://sourceforge.net/p/chebi/curator-requests/ - -## OBA - - - id: oba - title: Ontology of Biological Attributes - description: A collection of biological attributes (traits) covering all kingdoms of life. Incorporates VT (vertebrate trait ontology) and TO (plant trait ontology). Extends PATO. - contact: - email: cjmungall@lbl.gov - label: Chris Mungall - jobs: - - id: http://build.berkeleybop.org/job/build-bio-attributes - type: DryRunBuild - termgenie: http://oba.termgenie.org - domain: phenotype - homepage: http://wiki.geneontology.org/index.php/Extensions/x-attribute - page: http://wiki.geneontology.org/index.php/Extensions/x-attribute - products: - - - id: oba.owl - id: oba.obo - -## HP - - - id: hp - alternativePrefix: HPO - description: The Human Phenotype Ontology is being developed to provide a structured and controlled vocabulary for the phenotypic features encountered in human hereditary and other disease. Our goal is to provide resource for the computational analysis of the human phenome, with a current focus on monogenic diseases listed in the Online Mendelian Inheritance in Man (OMIM) database, for which annotations are also provided. - domain: phenotype - homepage: http://www.human-phenotype-ontology.org/ - contact: - email: peter.robinson@charite.de - label: Peter Robinson - products: - - id: hp.owl - - id: hp.obo - taxon: - id: NCBITaxon:9606 - label: Homo sapiens - title: human phenotype ontology - tracker: https://sourceforge.net/p/obo/human-phenotype-requests/ - termgenie: http://hp.termgenie,org - -## MP - - - id: mp - title: Mammalian phenotype - description: The Mammalian Phenotype Ontology is under development as a community effort to provide standard terms for annotating mammalian phenotypic data. - homepage: http://www.informatics.jax.org/searches/MP_form.shtml - contact: - email: pheno@jax.org - label: JAX phenotype list - domain: phenotype - products: - - id: mp.owl - - id: mp.obo - taxon: - id: NCBITaxon:10088 - label: Mus - tracker: https://sourceforge.net/p/obo/mammalian-phenotype-requests/ - termgenie: http://mp.termgenie.org - -## FBbt - - - id: fbbt - preferredPrefix: FBbt + ontology_purl: http://purl.obolibrary.org/obo/chebi.obo + - id: chebi.owl.gz + ontology_purl: http://purl.obolibrary.org/obo/chebi.owl.gz + title: chebi, compressed owl + - id: chebi/chebi_lite.obo + ontology_purl: http://purl.obolibrary.org/obo/chebi/chebi_lite.obo + title: chebi_lite, no syns or xrefs + - id: chebi/chebi_core.obo + ontology_purl: http://purl.obolibrary.org/obo/chebi/chebi_core.obo + title: chebi_core, no xrefs + publications: + - id: http://europepmc.org/article/MED/26467479 + title: 'ChEBI in 2016: Improved services and an expanding collection of metabolites.' + review: + date: 2010 + title: Chemical Entities of Biological Interest + tracker: https://github.com/ebi-chebi/ChEBI/issues + twitter: chebit + usages: + - description: Rhea uses CHEBI to annotate reaction participants + examples: + - description: Query for all usages of CHEBI:29748 (chorismate) + url: https://www.rhea-db.org/searchresults?q=CHEBI:29748 + user: https://www.rhea-db.org/ + - description: ZFIN uses CHEBI to annotate experiments + examples: + - description: A curated zebrafish experiment involving exposure to (5Z,8Z,14Z)-11,12-dihydroxyicosatrienoic + acid (CHEBI:63969) + url: http://zfin.org/action/expression/experiment?id=ZDB-EXP-190627-10 + user: http://zfin.org + - DO wiki: http://diseaseontology.sourceforge.net/ + activity_status: active + browsers: + - label: DO + title: DO Browser + url: http://www.disease-ontology.org/ + build: + infallible: 1 + method: obo2owl + source_url: https://raw.githubusercontent.com/DiseaseOntology/HumanDiseaseOntology/master/src/ontology/doid.obo contact: - email: djs93@gen.cam.ac.uk - label: David Osumi-Sutherland - description: A structured controlled vocabulary of the anatomy of Drosophila melanogaster - domain: anatomy - homepage: " http://purl.obolibrary.org/obo/fbbt" + email: lynn.schriml@gmail.com + github: lschriml + label: Lynn Schriml + description: An ontology for describing the classification of human diseases organized + by etiology. + domain: disease + facebook: https://www.facebook.com/diseaseontology + homepage: http://www.disease-ontology.org + id: doid + in_foundry_order: 1 + layout: ontology_detail + license: + label: CC0 1.0 Universal + logo: http://mirrors.creativecommons.org/presskit/buttons/80x15/png/cc-zero.png + url: https://creativecommons.org/publicdomain/zero/1.0/ + ontology_purl: http://purl.obolibrary.org/obo/doid.owl products: - - id: fbbt.owl - taxon: - id: NCBITaxon:7227 - label: Drosophila - title: Drosophila gross anatomy - tracker: http://purl.obolibrary.org/obo/fbbt/tracker - -## NCBITaxon -# -# - id: ncbitaxon -# preferredPrefix: NCBITaxon -# title: NCBI organismal classification -# contact: -# email: obo-taxonomy@lists.sourceforge.net -# label: obo-taxonomy-list -# description: The NCBITaxon ontology is an automatic translation of the NCBI taxonomy (a taxonomic classification of living organisms and associated artifacts) database into obo/owl. -# source: http://www.ncbi.nlm.nih.gov/taxonomy -# wasDerivedFrom: ftp://ftp.ebi.ac.uk/pub/databases/taxonomy/taxonomy.dat -# createdWith: http://owltools.googlecode.com/ -# domain: taxonomy -# homepage: http://www.obofoundry.org/wiki/index.php/NCBITaxon:Main_Page -# page: http://www.ncbi.nlm.nih.gov/taxonomy -# isInferred: true -# jobs: -# - id: http://build.berkeleybop.org/job/build-ncbitaxon/ -# type: ReleaseBuild -# products: -# - id: ncbitaxon.owl -# - id: ncbitaxon/subsets/taxslim.owl - - -## FMA - - - id: fma - contact: - email: mejino@comcast.net - label: Onard Mejino - description: Obo format translation of the FMA, omitting all relationships other than is_a, part_of and has_part. Future versions of fma_obo will include more relationships - domain: anatomy - homepage: http://sig.biostr.washington.edu/projects/fm/index.html - page: http://en.wikipedia.org/wiki/Foundational_Model_of_Anatomy + - id: doid.owl + ontology_purl: http://purl.obolibrary.org/obo/doid.owl + title: Disease Ontology, OWL format. This file contains DO's is_a asserted hierarchy + plus equivalent axioms to other OBO Foundry ontologies. + - id: doid.obo + ontology_purl: http://purl.obolibrary.org/obo/doid.obo + title: Disease Ontology, OBO format. This file omits the equivalent axioms. publications: - - id: http://www.ncbi.nlm.nih.gov/pubmed/18688289 - - id: http://www.ncbi.nlm.nih.gov/pubmed/18360535 - - id: http://www.ncbi.nlm.nih.gov/pubmed/16779026 - products: - - id: fma.owl + - id: http://www.ncbi.nlm.nih.gov/pubmed/?term=25348409 + title: 'Disease Ontology 2015 update: an expanded and updated database of human + diseases for linking biomedical knowledge through disease data' + review: + date: 2015 + document: + label: PDF + link: https://drive.google.com/open?id=0B8vqEgF1N0NIZ082U2JETHlSTGs taxon: id: NCBITaxon:9606 label: Homo sapiens - title: Foundational Model of Anatomy (subset) - tracker: https://sourceforge.net/p/obo/foundational-model-of-anatomy-fma-requests/ - -## IAO - - - id: iao + title: Human Disease Ontology + tracker: https://github.com/DiseaseOntology/HumanDiseaseOntology/issues + twitter: diseaseontology + usages: + - description: MGI disease model annotations use DO + examples: + - description: Human genes and mouse homology associated with nail diseases + url: http://www.informatics.jax.org/disease/DOID:4123 + user: http://www.informatics.jax.org/disease + - activity_status: active contact: - email: alanruttenberg@gmail.com - label: Alan Ruttenberg - description: "The Information Artifact Ontology (IAO) is a new ontology of information entities, originally driven by work by the OBI digital entity and realizable information entity branch. " - domain: information - homepage: http://purl.obolibrary.org/obo/iao - page: http://code.google.com/p/information-artifact-ontology/ - products: - - id: iao.owl - - - id: iao/d-acts.owl - title: ontology of document acts - description: "An ontology based on a theory of document acts describing what people can do with documents" - contact: - email: mbrochhausen@gmail.com - label: Mathias Brochhausen - - title: Information Artifact Ontology - tracker: http://code.google.com/p/information-artifact-ontology/issues/list - -## RS - - - id: rs - contact: - email: rnigam@mcw.edu - label: Rajni Nigam - description: Ontology of rat strains - homepage: http://rgd.mcw.edu/ - publications: - - id: http://www.ncbi.nlm.nih.gov/pubmed/24267899 - products: - - id: rs.owl - title: Rat Strain Ontology - -## DPO - - - id: fbcv - title: Drosophila Phenotype Ontology - preferredPrefix: FBcv - products: - - id: fbcv.owl - contact: - email: djs93@gen.cam.ac.uk - label: David Osumi-Sutherland - description: "An ontology of commonly encountered and/or high level Drosophila phenotypes. It has significant formalisation - utilising terms from GO, CL, PATO and the Drosophila anatomy ontology. It has been used by FlyBase for > 159000 annotations of phenotype" - homepage: http://www.flybase.org/ - -## PATO - - - id: pato - title: Phenotypic quality - description: Phenotypic qualities (properties). This ontology can be used in conjunction with other ontologies such as GO or anatomical ontologies to refer to phenotypes. Examples of qualities are red, ectopic, high temperature, fused, small, edematous and arrested. - contact: - email: geg18@aber.ac.uk - label: George Gkoutos - domain: phenotype - homepage: http://obofoundry.org/wiki/index.php/PATO:Main_Page - repository: https://github.com/gkoutos/pato - tracker: https://github.com/gkoutos/pato/issues - jobs: - - id: http://build.berkeleybop.org/job/build-pato - type: DryRunBuild - products: - - id: pato.owl - - id: pato.obo - - - build: {checkout: 'git clone https://github.com/AgriculturalSemantics/agro.git', - infallible: 1, method: vcs, path: ., system: git} - contact: {email: c.aubert@cgiar.org, github: celineaubert, label: Celine Aubert} - dependencies: - - {id: envo} - - {id: uo} - - {id: ro} - - {id: pato} - - {id: chebi} - - {id: iao} - description: Ontology of agronomic practices, agronomic techniques, and agronomic - variables used in agronomic experiments - domain: agronomy - homepage: https://github.com/AgriculturalSemantics/agro - id: agro - jobs: - - {id: 'https://travis-ci.org/AgriculturalSemantics/agro', type: travis-ci} + contact: mellybelly + email: haendel@ohsu.edu + label: Melissa Haendel + description: NCI Thesaurus (NCIt)is a reference terminology that includes broad + coverage of the cancer domain, including cancer related diseases, findings and + abnormalities. The NCIt OBO Edition aims to increase integration of the NCIt with + OBO Library ontologies. NCIt OBO Edition releases should be considered experimental. + homepage: https://github.com/NCI-Thesaurus/thesaurus-obo-edition + id: ncit layout: ontology_detail - license: {label: CC-BY, logo: 'http://mirrors.creativecommons.org/presskit/buttons/80x15/png/by.png', - url: 'http://creativecommons.org/licenses/by/3.0/'} - mailing_list: https://groups.google.com/group/agronomyOntology - ontology_purl: http://purl.obolibrary.org/obo/agro.owl + license: + label: CC-BY 4.0 + logo: http://mirrors.creativecommons.org/presskit/buttons/80x15/png/by.png + url: https://creativecommons.org/licenses/by/4.0/ + ontology_purl: http://purl.obolibrary.org/obo/ncit.owl products: - - {id: agro.owl, ontology_purl: 'http://purl.obolibrary.org/obo/agro.owl'} - title: Agronomy Ontology - tracker: https://github.com/AgriculturalSemantics/agro/issues/ \ No newline at end of file + - description: A direct transformation of the standard NCIt content using OBO-style + term and ontology IRIs and annotation properties. + id: ncit.owl + ontology_purl: http://purl.obolibrary.org/obo/ncit.owl + title: NCIt OBO Edition OWL format + - id: ncit.obo + ontology_purl: http://purl.obolibrary.org/obo/ncit.obo + title: NCIt OBO Edition OBO format + - description: This version replaces NCIt terms with direct references to terms + from other domain-specific OBO Library ontologies (e.g. cell types, cellular + components, anatomy), supporting cross-ontology reasoning. The current release + incorporates CL (cell types) and Uberon (anatomy). + id: ncit/ncit-plus.owl + mireots_from: uberon + ontology_purl: http://purl.obolibrary.org/obo/ncit/ncit-plus.owl + title: NCIt Plus + - description: This is a subset extracted from NCIt Plus, based on the [NCIt Neoplasm + Core value set](https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/Neoplasm/About_Core.html) + as a starting point. + id: ncit/neoplasm-core.owl + ontology_purl: http://purl.obolibrary.org/obo/ncit/neoplasm-core.owl + title: NCIt Plus Neoplasm Core + title: NCI Thesaurus OBO Edition + tracker: https://github.com/NCI-Thesaurus/thesaurus-obo-edition/issues diff --git a/ols-apps/ols-config-importer/src/main/resources/ols-config.yaml b/ols-apps/ols-config-importer/src/main/resources/ols-config.yaml index 24f08cab..07f6971c 100644 --- a/ols-apps/ols-config-importer/src/main/resources/ols-config.yaml +++ b/ols-apps/ols-config-importer/src/main/resources/ols-config.yaml @@ -2,7 +2,7 @@ ontologies: -## EFO + ## EFO - id: efo preferredPrefix: EFO @@ -12,8 +12,10 @@ ontologies: homepage: http://www.ebi.ac.uk/efo mailing_list: efo-users@lists.sourceforge.net definition_property: + - http://purl.obolibrary.org/obo/IAO_0000115 - http://www.ebi.ac.uk/efo/definition synonym_property: + - http://www.geneontology.org/formats/oboInOwl#hasExactSynonym - http://www.ebi.ac.uk/efo/alternative_term hierarchical_property: - http://purl.obolibrary.org/obo/BFO_0000050 @@ -24,318 +26,9 @@ ontologies: - http://www.ebi.ac.uk/efo/EFO_ reasoner: OWL2 oboSlims: false - ontology_purl : http://www.ebi.ac.uk/efo/efo.owl - - -## ORDO - - - id: ordo - preferredPrefix: ORDO - description: "The Orphanet Rare Disease ontology (ORDO) is jointly developed by Orphanet and the EBI to provide a structured vocabulary for rare diseases capturing relationships between diseases, genes and other relevant features which will form a useful resource for the computational analysis of rare diseases. - It derived from the Orphanet database (www.orpha.net ) , a multilingual database dedicated to rare diseases populated from literature and validated by international experts. - It integrates a nosology (classification of rare diseases), relationships (gene-disease relations, epiemological data) and connections with other terminologies (MeSH, UMLS, MedDRA),databases (OMIM, UniProtKB, HGNC, ensembl, Reactome, IUPHAR, Geantlas) or classifications (ICD10)." - title: Orphanet Rare Disease Ontolog - uri: http://www.orpha.net/ontology/orphanet.owl - definition_property: - - http://www.ebi.ac.uk/efo/definition - synonym_property: - - http://www.ebi.ac.uk/efo/alternative_term - hidden_property: - - http://www.ebi.ac.uk/efo/has_flag - hierarchical_property: - - http://purl.obolibrary.org/obo/BFO_0000050 - base_uri: - - http://www.orpha.net/ORDO/Orphanet_ - reasoner: OWL2 - oboSlims: false - ontology_purl : http://www.orphadata.org/data/ORDO/ordo_orphanet.owl - -## TEDDY - - - id: teddy - preferredPrefix: TEDDY - description: "TEDDY is an ontology for dynamical behaviours, observable dynamical phenomena, and control elements of bio-models and biological systems in Systems and Synthetic Biology." - title: "Terminology for Description of Dynamics" - uri: http://identifiers.org/teddy/ - definition_property: - - http://www.w3.org/2004/02/skos/core#definition - synonym_property: - - http://www.w3.org/2004/02/skos/core#altLabel - base_uri: - - http://identifiers.org/teddy/TEDDY_ - oboSlims: false - ontology_purl : http://www.biomodels.net/teddy/TEDDY - -## EDAM - - - id: edam - preferredPrefix: EDAM - title: EDAM bioinformatics ontology - uri: http://edamontology.org - description: "EDAM is a simple ontology of well established, familiar concepts that are prevalent within bioinformatics, including types of data and data identifiers, data formats, operations and topics. EDAM provides a set of terms with synonyms and definitions - organised into an intuitive hierarchy for convenient use." - homepage: http://edamontology.org - mailing_list: edam@elixir-dk.org - definition_property: - - http://www.geneontology.org/formats/oboInOwl#hasDefinition - synonym_property: - - http://www.geneontology.org/formats/oboInOwl#hasExactSynonym - base_uri: - - http://edamontology.org/data_ - - http://edamontology.org/identifier_ - - http://edamontology.org/topic_ - - http://edamontology.org/format_ - oboSlims: false - ontology_purl : http://edamontology.org/EDAM.owl ### This is the download URL - -## CCO - - - id: cco - preferredPrefix: CCO - title: Cell Cycle Ontology - uri: http://purl.obolibrary.org/obo/cco - description: "The Cell Cycle Ontology extends existing ontologies for cell cycle knowledge building a resource that integrates and manages knowledge about the cell cycle components and regulatory aspects." - homepage: http://www.semantic-systems-biology.org/apo - mailing_list: vladimir.n.mironov@gmail.com - definition_property: - - http://purl.obolibrary.org/obo/IAO_0000115 - synonym_property: - - http://www.geneontology.org/formats/oboInOwl#hasExactSynonym - base_uri: - - http://purl.obolibrary.org/obo/CCO_ - oboSlims: true - ontology_purl : http://www.bio.ntnu.no/ontology/CCO/cco.owl ### This is the download URL - -## CMPO - - - id: cmpo - preferredPrefix: CMPO - title: Cellular Microscopy Phenotype Ontology - uri: http://www.ebi.ac.uk/cmpo - description: "CMPO is a species neutral ontology for describing general phenotypic observations relating to the whole cell, cellular components, cellular processes and cell populations." - homepage: http://www.ebi.ac.uk/cmpo - mailing_list: jupp@ebi.ac.uk - definition_property: - - http://purl.obolibrary.org/obo/IAO_0000115 - synonym_property: - - http://www.geneontology.org/formats/oboInOwl#hasExactSynonym - base_uri: - - http://www.ebi.ac.uk/cmpo/CMPO_ - oboSlims: true - reasoner: EL - ontology_purl : http://www.ebi.ac.uk/cmpo/cmpo.owl ### This is the download URL - -## eNanoMapper - - - id: enm - preferredPrefix: ENM - title: eNanoMapper Ontology - uri: http://purl.enanomapper.org/onto/enanomapper.owl - description: "The eNanoMapper project (www.enanomapper.net) is creating a pan-European computational infrastructure for toxicological data management for ENMs, based on semantic web standards and ontologies. > This ontology is an application ontology targeting the f\ -ull domain of nanomaterial safety assessment. It re-uses several other ontologies including the NPO, CHEMINF, ChEBI, and ENVO. " - homepage: http://www.enanomapper.net/ - mailing_list: https://github.com/enanomapper/ontologies - definition_property: - - http://www.ebi.ac.uk/efo/definition - - http://purl.obolibrary.org/obo/IAO_0000115 - - http://purl.bioontology.org/ontology/npo#definition - - http://purl.obolibrary.org/obo#Definition - - http://purl.org/dc/elements/1.1/description - synonym_property: - - http://www.ebi.ac.uk/efo/alternative_term - - http://purl.obolibrary.org/obo/IAO_0000118 - hierarchical_property: - - http://www.bioassayontology.org/bao#BAO_0090002 - - http://purl.bioontology.org/ontology/npo#part_of - base_uri: - - http://purl.enanomapper.org/onto/ENM_ - oboSlims: false - ontology_purl : http://purl.enanomapper.net/onto/enanomapper.owl - -## ATOL - - - id: atol - preferredPrefix: ATOL - title: Animal Trait Ontology for Livestock - uri: file:/C:/Lea/ontologies/versions%20ATOL/atol_v3.0.obo - description: "ATOL (Animal Trait Ontology for Livestock) is an ontology of characteristics defining phenotypes of livestock in their environment (EOL). ATOL aims to: - provide a reference ontology of phenotypic traits of farm animals for the international scientific\ - and educational - communities, farmers, etc.; - deliver this reference ontology in a language which can be used by computers in order to support database management, semantic analysis and modeling; - represent traits as generic as possible for livestock vertebrates; - \ -make the ATOL ontology as operational as possible and closely related to measurement techniques; - structure the ontology in relation to animal production." - homepage: http://www.atol-ontology.com - mailing_list: pylebail@rennes.inra.fr - definition_property: - - "file:/C:/Lea/ontologies/versions%20ATOL/atol_v3.0.obo#definition" - synonym_property: - - "file:/C:/Lea/ontologies/versions%20ATOL/atol_v3.0.obo#synonymExact" - base_uri: - - http://purl.org/obo/owlATOL_ - oboSlims: false - ontology_purl : http://www.atol-ontology.com/public/telechargement/atol_v6.owl - -## EOL - - - id: eol - preferredPrefix: EOL - title: Environment Ontology for Livestock - uri: file:/C:/Lea/ontologies/environnement/leo.obo - description: "L'ontologie EOL décrit les conditions d'environnement des élevages d'animaux domestiques. Elle décrit plus particulièrement les modalités de l'alimentation, de l'environnement, de la structure des élevages et des systèmes d'élevage" - homepage: http://www.atol-ontology.com - mailing_list: pylebail@rennes.inra.fr - definition_property: - - "file:/C:/Lea/ontologies/environnement/leo.obo#definition" - synonym_property: - - "file:/C:/Lea/ontologies/environnement/leo.obo#synonymExact" - base_uri: - - http://purl.org/obo/owlEOL_ - ontology_purl : http://www.atol-ontology.com/ontology/20082013/eol_v4__collaboration.owl - -## LBO - - - id: lbo - preferredPrefix: LBO - title: Livestock Breed Ontology - uri: http://purl.obolibrary.org/obo/lbo - description: "A vocabulary for cattle, chicken, horse, pig, and sheep breeds." - homepage: http://bioportal.bioontology.org/ontologies/LBO - mailing_list: jreecy@iastate.edu - definition_property: - - "file:/C:/Lea/ontologies/environnement/leo.obo#definition" - synonym_property: - - "file:/C:/Lea/ontologies/environnement/leo.obo#synonymExact" - base_uri: - - http://purl.org/obo/owlEOL_ - oboSlims: false - ontology_purl : http://data.bioontology.org/ontologies/LBO/submissions/2/download?apikey=0911d614-1dd4-41c0-afd4-9f3df0fc70be - - -## AncestrO - - - id: ancestro - preferredPrefix: ancestro - title: Ancestry Ontology - uri: http://www.ebi.ac.uk/ancestro - description: "The Ancestry Ontology (Ancestro) provides a systematic description of the ancestry concepts used in the NHGRI-EBI Catalog of published genome-wide association studies. It includes a list of countries, regions and major areas (essentially continents), as well as a fairly exhaustive list of country- or region-specific ancestral groups, uncategorised ancestral groups and population isolates." - homepage: https://github.com/EBISPOT/ancestro/ - mailing_list: - definition_property: - - http://purl.obolibrary.org/obo/IAO_0000115 - synonym_property: - - http://purl.obolibrary.org/obo/IAO_0000118 - hierarchical_property: - - http://purl.obolibrary.org/obo/BFO_0000050 - base_uri: - - http://www.ebi.ac.uk/ancestro/ancestro_ - oboSlims: false - reasoner: OWL2 - ontology_purl : https://raw.githubusercontent.com/EBISPOT/ancestro/master/ancestro.owl - -## GAZ overide - - - id: gaz - is_foundry: true - ontology_purl: http://purl.obolibrary.org/obo/gaz.obo - reasoner: EL - hierarchical_property: - - http://purl.obolibrary.org/obo/BFO_0000050 - - http://purl.obolibrary.org/obo/RO_0001025 - - http://purl.obolibrary.org/obo/RO_0002151 - - http://purl.obolibrary.org/obo/RO_0002376 - - http://purl.obolibrary.org/obo/RO_0002377 - - http://purl.obolibrary.org/obo/UBREL_0000001 - - http://purl.obolibrary.org/obo/RO_0002219 - - http://purl.obolibrary.org/obo/RO_0002379 - - http://purl.obolibrary.org/obo/RO_0002378 - - http://purl.obolibrary.org/obo/RO_0002090 - - http://purl.obolibrary.org/obo/TEMP#proper_partof - - http://purl.obolibrary.org/obo/TEMP#overlapped_by - - http://purl.obolibrary.org/obo/RO_0002131 - - http://purl.obolibrary.org/obo/RO_0002220 - -## BTO overide - - - id: bto - is_foundry: true - ontology_purl: http://purl.obolibrary.org/obo/bto.owl - hierarchical_property: - - http://purl.obolibrary.org/obo/BFO_0000050 - - http://purl.obolibrary.org/obo/bto#develops_from - - http://purl.obolibrary.org/obo/bto#related_to - -## ProbOnto - - - id: probonto - preferredPrefix: probonto - title: Probability Ontology test - uri: http://www.probonto.org/ontology - description: "ProbOnto, is an ontology-based knowledge base of probability distributions, featuring more than eighty uni- and multivariate distributions with their defining functions, characteristics, relationships and reparameterisation formulas." - homepage: http://probonto.org - mailing_list: probonto.dev@gmail.com - definition_property: - - http://www.probonto.org/core#0000028 - base_uri: - - http://www.probonto.org/core - oboSlims: false - ontology_purl : https://raw.githubusercontent.com/probonto/ontology/master/probonto4ols.owl - - -## PRIDE ontology - - - id: pride - preferredPrefix: PRIDE - title: PRIDE Controlled Vocabulary - uri: http://purl.obolibrary.org/obo/pride_cv.obo - description: "The PRIDE PRoteomics IDEntifications (PRIDE) database is a centralized, standards compliant, public data repository for proteomics data, including protein and peptide identifications, post-translational modifications and supporting spectral evidence." - homepage: https://github.com/PRIDE-Utilities/pride-ontology - mailing_list: https://github.com/PRIDE-Utilities/pride-ontology/issues - definition_property: - - http://purl.obolibrary.org/obo/def - synonym_property: - - http://www.geneontology.org/formats/oboInOWL#hasExactSynonym - base_uri: - - http://purl.obolibrary.org/obo/PRIDE_ - hierarchical_property: - - http://purl.obolibrary.org/obo/pride_cv.obo#part_of - oboSlims: false - ontology_purl : https://raw.githubusercontent.com/PRIDE-Utilities/pride-ontology/master/pride_cv.obo - -## BTO overide - - - id: ncbitaxon - is_foundry: true - ontology_purl: http://purl.obolibrary.org/obo/ncbitaxon.owl - reasoner: NONE - -## ReXo - - - id: rexo - preferredPrefix: ReXO - title: Regulation of Gene Expression Ontology - uri: http://www.bio.ntnu.no/ontology/ReXO/rexo.owl - description: "Regulation of Gene Expression" - homepage: http://www.semantic-systems-biology.org/apo - mailing_list: vladimir.n.mironov@gmail.com - label_property: http://www.w3.org/2004/02/skos/core#prefLabel - definition_property: - - http://www.w3.org/2004/02/skos/core#definition - synonym_property: - - http://www.geneontology.org/formats/oboInOwl#hasExactSynonym - base_uri: - - http://purl.obolibrary.org/obo/ReXO_ - oboSlims: true - ontology_purl : http://www.bio.ntnu.no/ontology/ReXO/rexo.owl ### This is the download URL + ontology_purl: http://www.ebi.ac.uk/efo/efo.owl -## molecular interaction - - - id: mi - is_foundry: true - ontology_purl: http://purl.obolibrary.org/obo/mi-test.owl - hierarchical_property: - - http://purl.obolibrary.org/obo/BFO_0000050 + ## overide for mondo -# - id: mondo -# ontology_purl: http://purl.obolibrary.org/obo/mondo.owl -# reasoner: EL -# -# - id: upheno -# is_foundry: true -# ontology_purl: https://raw.githubusercontent.com/obophenotype/upheno/master/upheno.owl -# reasoner: EL + - id: mondo + ontology_purl: http://purl.obolibrary.org/obo/mondo.obo diff --git a/ols-apps/ols-indexer/Dockerfile b/ols-apps/ols-indexer/Dockerfile new file mode 100644 index 00000000..3cc38e13 --- /dev/null +++ b/ols-apps/ols-indexer/Dockerfile @@ -0,0 +1,17 @@ + +FROM maven:3.6-jdk-8 AS build +RUN mkdir /opt/ols +COPY . /opt/ols/ +COPY build-fix/. /root/.m2/repository/ +RUN cd /opt/ols && ls && mvn clean package -DskipTests + +FROM openjdk:8-jre-alpine +RUN apk add bash +COPY --from=build /opt/ols/ols-apps/ols-indexer/target/ols-indexer.jar /opt/ols-indexer.jar +EXPOSE 8080 + +ENV OLS_HOME /mnt/ + +ENTRYPOINT ["java", "-jar", "/opt/ols-indexer.jar"] + + diff --git a/ols-apps/ols-loading-app/pom.xml b/ols-apps/ols-indexer/pom.xml similarity index 83% rename from ols-apps/ols-loading-app/pom.xml rename to ols-apps/ols-indexer/pom.xml index e5225086..fe51f476 100644 --- a/ols-apps/ols-loading-app/pom.xml +++ b/ols-apps/ols-indexer/pom.xml @@ -1,17 +1,15 @@ - + uk.ac.ebi.spot ols-parent - 3.1.1 + 3.2.1-SNAPSHOT ../../ols-parent/pom.xml 4.0.0 - ols-loading-app + ols-indexer @@ -68,7 +66,6 @@ org.springframework.boot spring-boot-maven-plugin - ${springboot.maven} @@ -76,4 +73,4 @@ - \ No newline at end of file + diff --git a/ols-apps/ols-loading-app/src/main/java/uk/ac/ebi/spot/ols/LoadingApplication.java b/ols-apps/ols-indexer/src/main/java/uk/ac/ebi/spot/ols/LoadingApplication.java similarity index 65% rename from ols-apps/ols-loading-app/src/main/java/uk/ac/ebi/spot/ols/LoadingApplication.java rename to ols-apps/ols-indexer/src/main/java/uk/ac/ebi/spot/ols/LoadingApplication.java index 163b6730..65a0988b 100644 --- a/ols-apps/ols-loading-app/src/main/java/uk/ac/ebi/spot/ols/LoadingApplication.java +++ b/ols-apps/ols-indexer/src/main/java/uk/ac/ebi/spot/ols/LoadingApplication.java @@ -8,12 +8,14 @@ import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Import; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; import org.springframework.data.neo4j.config.EnableNeo4jRepositories; import org.springframework.data.neo4j.core.GraphDatabase; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import uk.ac.ebi.spot.ols.service.OntologyRepositoryService; +import uk.ac.ebi.spot.ols.config.OntologyLoadingConfiguration; import uk.ac.ebi.spot.ols.model.OntologyDocument; import uk.ac.ebi.spot.ols.service.FileUpdatingService; import uk.ac.ebi.spot.ols.service.OntologyIndexingService; @@ -40,11 +42,7 @@ @EnableMongoRepositories(basePackages = "uk.ac.ebi.spot.ols.repository.mongo") public class LoadingApplication implements CommandLineRunner { - private Logger log = LoggerFactory.getLogger(getClass()); - - public Logger getLog() { - return log; - } + private final Logger logger = LoggerFactory.getLogger(LoadingApplication.class); @Autowired OntologyRepositoryService ontologyRepositoryService; @@ -58,10 +56,12 @@ public Logger getLog() { @Autowired MailService mailService; - private static String [] ontologies = {}; + private static String [] forcedOntologies = {}; private static String email; + private static String [] deleteOntologies = {}; + private static boolean offline = false; @@ -69,23 +69,25 @@ public Logger getLog() { public void run(String... args) throws Exception { int parseArgs = parseArguments(args); + System.setProperty("entityExpansionLimit", "10000000"); Collection updatedOntologies = new HashSet<>(); + Map failingOntologies= new HashMap<>(); ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); executor.setCorePoolSize(10); executor.setMaxPoolSize(20); - executor.setQueueCapacity(200); + executor.setQueueCapacity(500); executor.initialize(); List allDocuments = new ArrayList(); - if (ontologies.length > 0) { + if (forcedOntologies.length > 0) { // get the ontologies forced to update - for (String ontologyName : ontologies) { - OntologyDocument document = ontologyRepositoryService.get(ontologyName); - if (document != null) { + for (String ontologyName : forcedOntologies) { + OntologyDocument document = ontologyRepositoryService.get(ontologyName); + if (document != null) { if (!offline) { // check these documents for updates allDocuments.add(ontologyRepositoryService.get(ontologyName)); @@ -98,6 +100,25 @@ public void run(String... args) throws Exception { } } } + else if (deleteOntologies.length > 0){ + // get the ontologies requested for deletion + for (String ontologyName : deleteOntologies) { + OntologyDocument document = ontologyRepositoryService.get(ontologyName); + + if (document != null) { + document.setStatus(Status.TOREMOVE); + ontologyRepositoryService.update(document); + } + else { + StringBuffer errorMessage = new StringBuffer("Could not detele ontology "); + errorMessage.append(ontologyName); + errorMessage.append(" as it doesn't exist in OLS"); + logger.warn(errorMessage.toString()); + failingOntologies.put(document.getOntologyId(), errorMessage.toString()); + } + + } + } else if (!offline){ // get all documents and check for updates allDocuments = ontologyRepositoryService.getAllDocuments(); @@ -105,7 +126,7 @@ else if (!offline){ CountDownLatch latch = new CountDownLatch(allDocuments.size()); FileUpdatingService service = new FileUpdatingService(ontologyRepositoryService, executor, latch); - service.checkForUpdates(allDocuments, fileUpdater, ontologies.length>0); + service.checkForUpdates(allDocuments, fileUpdater, forcedOntologies.length>0); // wait for ontologies to have been checked latch.await(); @@ -116,19 +137,38 @@ else if (!offline){ boolean haserror = false; // if force loading - + long start = System.currentTimeMillis(); StringBuilder exceptions = new StringBuilder(); - if (ontologies.length > 0) { - for (String ontologyName : ontologies) { + if (forcedOntologies.length > 0) { + for (String ontologyName : forcedOntologies) { OntologyDocument document = ontologyRepositoryService.get(ontologyName); if (document != null) { try { ontologyIndexingService.indexOntologyDocument(document); updatedOntologies.add(document.getOntologyId()); - } catch (Exception e) { - getLog().error("Application failed creating indexes for " + document.getOntologyId() + ": " + e.getMessage()); + } catch (Throwable t) { + logger.error("Application failed creating indexes for " + + document.getOntologyId() + ": " + t.getMessage(), t); haserror = true; + failingOntologies.put(document.getOntologyId(), t.getMessage()); + } + } + } + } + else if (deleteOntologies.length > 0){ + for (String ontologyName : deleteOntologies) { + OntologyDocument document = ontologyRepositoryService.get(ontologyName); + if (document != null) { + try { + ontologyIndexingService.removeOntologyDocumentFromIndex(document); + ontologyRepositoryService.delete(document); + updatedOntologies.add(document.getOntologyId()); + } catch (Throwable t) { + logger.error("Application failed deleting indexes for " + document.getOntologyId() + ": " + + t.getMessage(), t); + haserror = true; + failingOntologies.put(document.getOntologyId(), t.getMessage()); } } } @@ -137,18 +177,24 @@ else if (!offline){ // otherwise load everything set TOLOAD for (OntologyDocument document : ontologyRepositoryService.getAllDocumentsByStatus(Status.TOLOAD)) { try { - ontologyIndexingService.indexOntologyDocument(document); - updatedOntologies.add(document.getOntologyId()); - } catch (Exception e) { - getLog().error("Application failed creating indexes for " + document.getOntologyId() + ": " + e.getMessage()); - exceptions.append(e.getMessage()); + boolean loadResult = ontologyIndexingService.indexOntologyDocument(document); + if (loadResult) + updatedOntologies.add(document.getOntologyId()); + else { + haserror = true; + failingOntologies.put(document.getOntologyId(), "An error occurred. Check logs."); + } + } catch (Throwable t) { + logger.error("Application failed creating indexes for " + document.getOntologyId() + ": " + + t.getMessage(), t); + exceptions.append(t.getMessage()); exceptions.append("\n"); haserror = true; + failingOntologies.put(document.getOntologyId(),t.getMessage()); } } } - Map failingOntologies= new HashMap<>(); for (OntologyDocument document : ontologyRepositoryService.getAllDocumentsByStatus(Status.FAILED)) { failingOntologies.put(document.getOntologyId(), document.getMessage()); } @@ -162,14 +208,14 @@ else if (!offline){ } - + long end = System.currentTimeMillis(); + logger.debug("Duration of indexing = " + (end - start)/1000/60 + " minutes."); if (haserror) { System.exit(1); } System.exit(0); } - private static int parseArguments(String[] args) { CommandLineParser parser = new GnuParser(); @@ -189,7 +235,7 @@ private static int parseArguments(String[] args) { else { // find -f option to see if we are to force load if (cl.hasOption("f") ) { - ontologies = cl.getOptionValues("f"); + forcedOntologies = cl.getOptionValues("f"); } offline = cl.hasOption("off"); @@ -201,6 +247,10 @@ private static int parseArguments(String[] args) { email = cl.getOptionValue("m"); } + if (cl.hasOption("d")){ + deleteOntologies = cl.getOptionValues("d"); + } + } } @@ -240,7 +290,13 @@ private static Options bindOptions() { "Send e-mail report"); mail.setRequired(false); options.addOption(mail); + + Option delete = new Option("d", "delete", true, "List the ontologies to be deleted"); + delete.setRequired(false); + options.addOption(delete); + return options; + } public static void main(String[] args) throws Exception { diff --git a/ols-apps/ols-loading-app/src/main/java/uk/ac/ebi/spot/ols/LoadingReport.java b/ols-apps/ols-indexer/src/main/java/uk/ac/ebi/spot/ols/LoadingReport.java similarity index 100% rename from ols-apps/ols-loading-app/src/main/java/uk/ac/ebi/spot/ols/LoadingReport.java rename to ols-apps/ols-indexer/src/main/java/uk/ac/ebi/spot/ols/LoadingReport.java diff --git a/ols-apps/ols-loading-app/src/main/java/uk/ac/ebi/spot/ols/LoadingReportPrinter.java b/ols-apps/ols-indexer/src/main/java/uk/ac/ebi/spot/ols/LoadingReportPrinter.java similarity index 95% rename from ols-apps/ols-loading-app/src/main/java/uk/ac/ebi/spot/ols/LoadingReportPrinter.java rename to ols-apps/ols-indexer/src/main/java/uk/ac/ebi/spot/ols/LoadingReportPrinter.java index 8867154b..30b4634c 100644 --- a/ols-apps/ols-loading-app/src/main/java/uk/ac/ebi/spot/ols/LoadingReportPrinter.java +++ b/ols-apps/ols-indexer/src/main/java/uk/ac/ebi/spot/ols/LoadingReportPrinter.java @@ -17,7 +17,7 @@ public static String getMessage (LoadingReport loadingReport) { builder.append("---------------------------\n"); } else { - builder.append("The following ontologies were sucessfully updated\n"); + builder.append("The following ontologies were successfully updated\n"); builder.append("-------------------------------------------------\n"); for (String name: loadingReport.getUpdatedOntologies()) { diff --git a/ols-apps/ols-loading-app/src/main/java/uk/ac/ebi/spot/ols/MailConfig.java b/ols-apps/ols-indexer/src/main/java/uk/ac/ebi/spot/ols/MailConfig.java similarity index 100% rename from ols-apps/ols-loading-app/src/main/java/uk/ac/ebi/spot/ols/MailConfig.java rename to ols-apps/ols-indexer/src/main/java/uk/ac/ebi/spot/ols/MailConfig.java diff --git a/ols-apps/ols-loading-app/src/main/java/uk/ac/ebi/spot/ols/MailService.java b/ols-apps/ols-indexer/src/main/java/uk/ac/ebi/spot/ols/MailService.java similarity index 100% rename from ols-apps/ols-loading-app/src/main/java/uk/ac/ebi/spot/ols/MailService.java rename to ols-apps/ols-indexer/src/main/java/uk/ac/ebi/spot/ols/MailService.java diff --git a/ols-apps/ols-loading-app/src/main/resources/application.properties b/ols-apps/ols-indexer/src/main/resources/application.properties similarity index 66% rename from ols-apps/ols-loading-app/src/main/resources/application.properties rename to ols-apps/ols-indexer/src/main/resources/application.properties index a28ac5c0..c516a550 100644 --- a/ols-apps/ols-loading-app/src/main/resources/application.properties +++ b/ols-apps/ols-indexer/src/main/resources/application.properties @@ -1,4 +1,5 @@ spring.main.show_banner=false +logging.file=ols-indexer.log logging.level.org.springframework=ERROR logging.level.ch.qos.logback.*=ERROR logging.level.org.semanticweb.elk.*=ERROR @@ -6,4 +7,6 @@ spring.data.mongodb.database ols mail.from spot-ontodev@ebi.ac.uk -obo.db.xrefs https://raw.githubusercontent.com/geneontology/go-site/master/metadata/db-xrefs.yaml \ No newline at end of file +obo.db.xrefs https://raw.githubusercontent.com/geneontology/go-site/master/metadata/db-xrefs.yaml + +annotationproperty.preferredroot.term=http://purl.obolibrary.org/obo/IAO_0000700 \ No newline at end of file diff --git a/ols-apps/ols-indexer/src/main/resources/logback-spring.xml b/ols-apps/ols-indexer/src/main/resources/logback-spring.xml new file mode 100644 index 00000000..ca97b7c4 --- /dev/null +++ b/ols-apps/ols-indexer/src/main/resources/logback-spring.xml @@ -0,0 +1,27 @@ + + + + + %d{ISO8601} %p %t %c{0}.%M - %m%n + utf8 + + + + ols-loading-app.log + false + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{35} %M - %msg%n + + + + + + + + + + + + + + diff --git a/ols-apps/ols-loading-app/src/test/java/resources/application.properties b/ols-apps/ols-indexer/src/test/java/resources/application.properties similarity index 100% rename from ols-apps/ols-loading-app/src/test/java/resources/application.properties rename to ols-apps/ols-indexer/src/test/java/resources/application.properties diff --git a/ols-apps/ols-loading-app/src/test/java/resources/file1.txt b/ols-apps/ols-indexer/src/test/java/resources/file1.txt similarity index 100% rename from ols-apps/ols-loading-app/src/test/java/resources/file1.txt rename to ols-apps/ols-indexer/src/test/java/resources/file1.txt diff --git a/ols-apps/ols-loading-app/src/test/java/uk/ac/ebi/spot/ols/FileUpdateTests.java b/ols-apps/ols-indexer/src/test/java/uk/ac/ebi/spot/ols/FileUpdateTests.java similarity index 94% rename from ols-apps/ols-loading-app/src/test/java/uk/ac/ebi/spot/ols/FileUpdateTests.java rename to ols-apps/ols-indexer/src/test/java/uk/ac/ebi/spot/ols/FileUpdateTests.java index 0729da47..641ba523 100644 --- a/ols-apps/ols-loading-app/src/test/java/uk/ac/ebi/spot/ols/FileUpdateTests.java +++ b/ols-apps/ols-indexer/src/test/java/uk/ac/ebi/spot/ols/FileUpdateTests.java @@ -34,7 +34,7 @@ public void testFileUpdater () { e.printStackTrace(); } - System.out.println(status.isNew()); + System.out.println(status.getLatestHash()); } diff --git a/ols-apps/ols-neo4j-app/pom.xml b/ols-apps/ols-neo4j-app/pom.xml index f3ae0c47..e79285c1 100644 --- a/ols-apps/ols-neo4j-app/pom.xml +++ b/ols-apps/ols-neo4j-app/pom.xml @@ -1,11 +1,9 @@ - + uk.ac.ebi.spot ols-parent - 3.1.1 + 3.2.1-SNAPSHOT ../../ols-parent/pom.xml 4.0.0 @@ -44,8 +42,6 @@ org.springframework.boot spring-boot-maven-plugin - ${springboot.maven} - diff --git a/ols-apps/ols-solr-app/pom.xml b/ols-apps/ols-solr-app/pom.xml index 2897f934..6b7b2510 100644 --- a/ols-apps/ols-solr-app/pom.xml +++ b/ols-apps/ols-solr-app/pom.xml @@ -1,11 +1,9 @@ - + uk.ac.ebi.spot ols-parent - 3.1.1 + 3.2.1-SNAPSHOT ../../ols-parent/pom.xml 4.0.0 @@ -35,7 +33,6 @@ org.springframework.boot spring-boot-maven-plugin - ${springboot.maven} diff --git a/ols-apps/pom.xml b/ols-apps/pom.xml index 6b39b9c3..39a10c1a 100644 --- a/ols-apps/pom.xml +++ b/ols-apps/pom.xml @@ -1,19 +1,30 @@ - + 4.0.0 uk.ac.ebi.spot ols-apps - 3.1.1 + 3.2.1-SNAPSHOT pom ols-solr-app - ols-loading-app + ols-indexer ols-config-importer ols-neo4j-app + + + nexus-release + Releases + https://www.ebi.ac.uk/spot/nexus/repository/maven-releases/ + + + nexus-snapshot + Snapshot + https://www.ebi.ac.uk/spot/nexus/repository/maven-snapshots/ + + - \ No newline at end of file + + diff --git a/ols-core/pom.xml b/ols-core/pom.xml index 95a1697f..e3b317ba 100644 --- a/ols-core/pom.xml +++ b/ols-core/pom.xml @@ -1,11 +1,9 @@ - + uk.ac.ebi.spot ols-parent - 3.1.1 + 3.2.1-SNAPSHOT ../ols-parent/pom.xml 4.0.0 diff --git a/ols-core/src/main/java/uk/ac/ebi/spot/ols/model/OntologyDocument.java b/ols-core/src/main/java/uk/ac/ebi/spot/ols/model/OntologyDocument.java index 0da843b7..4e9f1c6f 100644 --- a/ols-core/src/main/java/uk/ac/ebi/spot/ols/model/OntologyDocument.java +++ b/ols-core/src/main/java/uk/ac/ebi/spot/ols/model/OntologyDocument.java @@ -34,7 +34,9 @@ public class OntologyDocument { private String localPath; private String version; + private String fileHash; + private int loadAttempts; private int numberOfTerms; private int numberOfProperties; @@ -42,12 +44,17 @@ public class OntologyDocument { private OntologyResourceConfig config; - public OntologyDocument(String ontologyId, Date updated, Status status, String message, String localPath, int numberOfTerms, int numberOfProperties, int numberOfIndividuals, OntologyResourceConfig config) { + public OntologyDocument(String ontologyId, Date updated, Status status, String message, String localPath, + String fileHash, int loadAttempts, + int numberOfTerms, int numberOfProperties, int numberOfIndividuals, + OntologyResourceConfig config) { this.ontologyId = ontologyId; this.updated = updated; this.status = status; this.message = message; this.localPath = localPath; + this.fileHash = fileHash; + this.loadAttempts = loadAttempts; this.numberOfTerms = numberOfTerms; this.numberOfProperties = numberOfProperties; this.numberOfIndividuals = numberOfIndividuals; @@ -60,7 +67,8 @@ public OntologyDocument() { } public OntologyDocument(String ontologyId, OntologyResourceConfig config) { - this(ontologyId, new Date(), Status.NOTLOADED, "No ontology loaded", null, 0,0,0, config); + this(ontologyId, new Date(), Status.NOTLOADED, "No ontology loaded", null,"",0, 0, + 0,0, config); } public String getVersion() { @@ -150,4 +158,20 @@ public String getLocalPath() { public void setLocalPath(String localPath) { this.localPath = localPath; } + + public String getFileHash() { + return fileHash; + } + + public void setFileHash(String fileHash) { + this.fileHash = fileHash; + } + + public int getLoadAttempts() { + return loadAttempts; + } + + public void setLoadAttempts(int n) { + this.loadAttempts = n; + } } diff --git a/ols-core/src/main/java/uk/ac/ebi/spot/ols/model/OntologyIndexer.java b/ols-core/src/main/java/uk/ac/ebi/spot/ols/model/OntologyIndexer.java index b6b356a8..65f5ad6d 100644 --- a/ols-core/src/main/java/uk/ac/ebi/spot/ols/model/OntologyIndexer.java +++ b/ols-core/src/main/java/uk/ac/ebi/spot/ols/model/OntologyIndexer.java @@ -20,6 +20,8 @@ public interface OntologyIndexer { void createIndex (OntologyLoader loader) throws IndexingException; + void dropIndex (String ontologyName) throws IndexingException; + void dropIndex (OntologyLoader loader) throws IndexingException; } diff --git a/ols-core/src/main/java/uk/ac/ebi/spot/ols/model/Status.java b/ols-core/src/main/java/uk/ac/ebi/spot/ols/model/Status.java index e60b29f6..875a1018 100644 --- a/ols-core/src/main/java/uk/ac/ebi/spot/ols/model/Status.java +++ b/ols-core/src/main/java/uk/ac/ebi/spot/ols/model/Status.java @@ -14,6 +14,10 @@ public enum Status { TOLOAD, LOADED, LOADING, - DOWNLOADING, FAILED + DOWNLOADING, + FAILED, + TOREMOVE, + REMOVED, + SKIP // after too many failed attempts } diff --git a/ols-core/src/main/java/uk/ac/ebi/spot/ols/service/FileUpdatingService.java b/ols-core/src/main/java/uk/ac/ebi/spot/ols/service/FileUpdatingService.java index 67cb64c4..d118c355 100644 --- a/ols-core/src/main/java/uk/ac/ebi/spot/ols/service/FileUpdatingService.java +++ b/ols-core/src/main/java/uk/ac/ebi/spot/ols/service/FileUpdatingService.java @@ -33,6 +33,8 @@ public class FileUpdatingService { private Logger log = LoggerFactory.getLogger(getClass()); + private static int MAX_LOAD_ATTEMPTS = 3; + public Logger getLog() { return log; } @@ -60,33 +62,55 @@ public void run() { getLog().info("Checking status of " + document.getOntologyId()); OntologyResourceConfig config = document.getConfig(); - // check if was previously failing boolean wasFailing = false; if (document.getStatus() != null) { - // - if (document.getStatus().equals(Status.FAILED) || document.getStatus().equals(Status.NOTLOADED)) { + if (document.getStatus().equals(Status.FAILED) || document.getStatus().equals(Status.NOTLOADED) || + document.getStatus().equals(Status.LOADING) || document.getStatus().equals(Status.DOWNLOADING)) { wasFailing = true; + getLog().debug(document.getOntologyId() + " + failed previously with status = " + document.getStatus()); + } else { + document.setLoadAttempts(0); } } + boolean skip = document.getStatus() == Status.SKIP; + + if(!skip) { + if(wasFailing) { + int newLoadAttempts = document.getLoadAttempts() + 1; + document.setLoadAttempts(newLoadAttempts); + + getLog().info(document.getOntologyId() + " has failed " + newLoadAttempts + " times out of " + MAX_LOAD_ATTEMPTS + " max"); - document.setStatus(Status.DOWNLOADING); - document.setUpdated(new Date()); - document.setMessage(""); - ontologyRepositoryService.update(document); + if(newLoadAttempts >= MAX_LOAD_ATTEMPTS) { + getLog().info(document.getOntologyId() + " failed too many times; skipping now and in future"); + document.setStatus(Status.SKIP); + skip = true; + } + } + } FileUpdater.FileStatus status = null; + try { - status = fileUpdateService.getFile(config.getNamespace(), config.getFileLocation()); - document.setLocalPath(status.getFile().getCanonicalPath()); - if (force || status.isNew() || wasFailing) { - document.setStatus(Status.TOLOAD); - document.setMessage(""); - } - else { - document.setStatus(Status.LOADED); + if(!skip) { + document.setStatus(Status.DOWNLOADING); + document.setUpdated(new Date()); document.setMessage(""); + ontologyRepositoryService.update(document); + + status = fileUpdateService.getFile(config.getNamespace(), config.getFileLocation()); + document.setLocalPath(status.getFile().getCanonicalPath()); + String fileHash = document.getFileHash(); + if (force || fileHash == null || !fileHash.equals(status.getLatestHash()) || wasFailing) { + document.setStatus(Status.TOLOAD); + document.setMessage(""); + } + else { + document.setStatus(Status.LOADED); + document.setMessage(""); + } } } catch (FileUpdateServiceException e) { @@ -97,7 +121,7 @@ public void run() { document.setStatus(Status.FAILED); } document.setMessage(e.getMessage()); - log.error("Error checking: " + config.getTitle() + e.getMessage()); + log.error("Error checking: " + config.getTitle() + e.getMessage(), e); } catch (IOException e) { if (document.getLoaded() == null) { document.setStatus(Status.NOTLOADED); @@ -112,6 +136,10 @@ public void run() { getLog().info("Status of " + document.getOntologyId() + " is " + document.getStatus()); document.setUpdated(new Date()); + + if(status != null) + document.setFileHash(status.getLatestHash()); + ontologyRepositoryService.update(document); latch.countDown(); } diff --git a/ols-core/src/main/java/uk/ac/ebi/spot/ols/service/OntologyIndexingService.java b/ols-core/src/main/java/uk/ac/ebi/spot/ols/service/OntologyIndexingService.java index 0d4a54ca..02b28d4a 100644 --- a/ols-core/src/main/java/uk/ac/ebi/spot/ols/service/OntologyIndexingService.java +++ b/ols-core/src/main/java/uk/ac/ebi/spot/ols/service/OntologyIndexingService.java @@ -13,6 +13,8 @@ */ public interface OntologyIndexingService { - void indexOntologyDocument(OntologyDocument document) throws IndexingException; + boolean indexOntologyDocument(OntologyDocument document) throws IndexingException; + + void removeOntologyDocumentFromIndex(OntologyDocument document) throws IndexingException; } diff --git a/ols-core/src/main/java/uk/ac/ebi/spot/ols/util/FileUpdater.java b/ols-core/src/main/java/uk/ac/ebi/spot/ols/util/FileUpdater.java index 8d06cf11..961f8f4a 100644 --- a/ols-core/src/main/java/uk/ac/ebi/spot/ols/util/FileUpdater.java +++ b/ols-core/src/main/java/uk/ac/ebi/spot/ols/util/FileUpdater.java @@ -120,23 +120,19 @@ public FileStatus getFile(String name, URI file) throws FileUpdateServiceExcepti // create one for the downloaded file - // if they are the same, the file hasn't changed - if (latestChecksum.equals(downloadChecksum)) { - return new FileStatus(latestFile, false); - } - else { + // if they are different, the file has changed + if (!latestChecksum.equals(downloadChecksum)) { // if they are different, copy the downloaded file to the latest file FileCopyUtils.copy(downloadFile, latestFile ); // update the latest file checksum writeChecksum(latestFileChecksum, downloadChecksum); - return new FileStatus(latestFile, true); } } else { FileCopyUtils.copy(downloadFile, latestFile ); writeChecksum(latestFileChecksum, downloadChecksum); - return new FileStatus(latestFile, true); } + return new FileStatus(latestFile, downloadChecksum); } catch (Exception e) { throw new FileUpdateServiceException("Failed to download file: " + e.getMessage(), e); @@ -240,20 +236,19 @@ private void writeChecksum (File fileCheck, String checksum) throws IOException public class FileStatus { private File file; - private boolean isNew; - + private String latestHash; - public FileStatus(File latestFile, boolean b) { + public FileStatus(File latestFile, String latestHash) { this.file = latestFile; - this.isNew = b; + this.latestHash = latestHash; } public File getFile() { return file; } - public boolean isNew() { - return isNew; + public String getLatestHash() { + return latestHash; } } diff --git a/ols-dependencies/pom.xml b/ols-dependencies/pom.xml index 58770624..2e50992f 100644 --- a/ols-dependencies/pom.xml +++ b/ols-dependencies/pom.xml @@ -1,11 +1,9 @@ - + uk.ac.ebi.spot ols-dependencies - 3.1.1 + 3.2.1-SNAPSHOT pom @@ -18,13 +16,13 @@ 4.0.0 - 3.1.1 - 3.5.2 - 1.3.8.4 - 0.4.2 + 5.1.14 + 1.4.3.517 + 0.5.0-SNAPSHOT 1.2 1.2.5.RELEASE 2.14.3 + 5.4.0 @@ -41,34 +39,38 @@ uk.ac.ebi.spot ols-core - ${ols.version} + ${project.version} uk.ac.ebi.spot ols-solr - ${ols.version} + ${project.version} uk.ac.ebi.spot ontology-tools - ${ols.version} + ${project.version} uk.ac.ebi.spot ols-web - ${ols.version} + ${project.version} uk.ac.ebi.spot ols-neo4j - ${ols.version} + ${project.version} uk.ac.ebi.spot ols-mongo - ${ols.version} + ${project.version} + + + uk.ac.ebi.spot + resource-usage + ${project.version} - net.sourceforge.owlapi @@ -76,14 +78,42 @@ ${owlapi.version} - com.hermit-reasoner + net.sourceforge.owlapi org.semanticweb.hermit ${HermiT.version} + + + net.sourceforge.owlapi + owlapi-apibinding + + + net.sourceforge.owlapi + owlapi-api + + + net.sourceforge.owlapi + owlapi-impl + + org.semanticweb.elk - elk-owlapi + elk-owlapi5 ${elk.version} + + + net.sourceforge.owlapi + owlapi-apibinding + + + net.sourceforge.owlapi + owlapi-api + + + net.sourceforge.owlapi + owlapi-impl + + commons-cli @@ -97,17 +127,35 @@ ${springboot.maven} - + + org.junit.jupiter + junit-jupiter-engine + ${junit.jupiter.version} + test + + + + org.junit.jupiter + junit-jupiter-api + ${junit.jupiter.version} + test + + + + org.junit.jupiter + junit-jupiter-params + ${junit.jupiter.version} + test + - - spring-milestones - Spring Milestones - http://repo.spring.io/milestone + maven-neo4j + Maven Neo4j + https://m2.neo4j.org/content/repositories/releases/ false @@ -128,7 +176,7 @@ neo4j Neo4j - http://m2.neo4j.org/ + http://m2.neo4j.org/content/repositories/releases/ @@ -136,5 +184,29 @@ jcenter https://jcenter.bintray.com/ + + sonatype-nexus-snapshots + https://oss.sonatype.org/content/repositories/snapshots/ + + false + + + true + + - \ No newline at end of file + + + + nexus-release + Releases + https://www.ebi.ac.uk/spot/nexus/repository/maven-releases/ + + + nexus-snapshot + Snapshot + https://www.ebi.ac.uk/spot/nexus/repository/maven-snapshots/ + + + + diff --git a/ols-mongo/pom.xml b/ols-mongo/pom.xml index fd03aa3d..f4bf409a 100644 --- a/ols-mongo/pom.xml +++ b/ols-mongo/pom.xml @@ -1,11 +1,9 @@ - + ols-parent uk.ac.ebi.spot - 3.1.1 + 3.2.1-SNAPSHOT ../ols-parent/pom.xml 4.0.0 @@ -33,6 +31,23 @@ org.springframework.data spring-data-rest-core + + + org.junit.jupiter + junit-jupiter-engine + + + + org.junit.jupiter + junit-jupiter-params + test + + + + org.junit.jupiter + junit-jupiter-api + test + \ No newline at end of file diff --git a/ols-mongo/src/main/java/uk/ac/ebi/spot/ols/service/MongoOntologyIndexingService.java b/ols-mongo/src/main/java/uk/ac/ebi/spot/ols/service/MongoOntologyIndexingService.java index ae26e3b1..0a413160 100644 --- a/ols-mongo/src/main/java/uk/ac/ebi/spot/ols/service/MongoOntologyIndexingService.java +++ b/ols-mongo/src/main/java/uk/ac/ebi/spot/ols/service/MongoOntologyIndexingService.java @@ -4,8 +4,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Import; import org.springframework.core.io.FileSystemResource; import org.springframework.stereotype.Component; + +import uk.ac.ebi.spot.ols.config.OntologyLoadingConfiguration; import uk.ac.ebi.spot.ols.config.OntologyResourceConfig; import uk.ac.ebi.spot.ols.exception.IndexingException; import uk.ac.ebi.spot.ols.exception.OntologyLoadingException; @@ -15,6 +18,7 @@ import uk.ac.ebi.spot.ols.model.OntologyDocument; import uk.ac.ebi.spot.ols.model.OntologyIndexer; import uk.ac.ebi.spot.ols.xrefs.DatabaseService; +import uk.ac.ebi.spot.usage.ResourceUsage; import java.text.SimpleDateFormat; import java.util.Collection; @@ -30,11 +34,7 @@ @Component public class MongoOntologyIndexingService implements OntologyIndexingService{ - private Logger log = LoggerFactory.getLogger(getClass()); - - public Logger getLog() { - return log; - } + private final Logger logger = LoggerFactory.getLogger(MongoOntologyIndexingService.class); @Autowired OntologyRepositoryService ontologyRepositoryService; @@ -44,19 +44,29 @@ public Logger getLog() { @Autowired DatabaseService databaseService; + + @Autowired + OntologyLoadingConfiguration ontologyLoadingConfiguration; @Override - public void indexOntologyDocument(OntologyDocument document) throws IndexingException { + public boolean indexOntologyDocument(OntologyDocument document) throws IndexingException { OntologyLoader loader = null; Collection classes; Collection properties; Collection individuals; String message = ""; - Status status = Status.FAILED; + Status status = Status.LOADING; + boolean result = false; + logger.trace("annotationproperty.preferredroot.term = " + + ontologyLoadingConfiguration.getPreferredRootTermAnnotationProperty()); + + ResourceUsage.logUsage(logger, "#### Monitoring ", document.getOntologyId() + + ":Before loading ontology", ":"); try { - loader = OntologyLoaderFactory.getLoader(document.getConfig(), databaseService); + loader = OntologyLoaderFactory.getLoader(document.getConfig(), databaseService, + ontologyLoadingConfiguration); if (document.getLocalPath() != null) { // if updated get local path, and set location to local file loader.setOntologyResource(new FileSystemResource(document.getLocalPath())); @@ -68,7 +78,7 @@ public void indexOntologyDocument(OntologyDocument document) throws IndexingExce // this means that file parsed, but had nothing in it, which is a bit suspect - indexing should fail until we undertand why/how this could happen if (classes.size() + properties.size() + individuals.size()== 0) { - getLog().error("A suspiciously small or zero classes or properties found in latest version of " + loader.getOntologyName() + ": Won't index!"); + logger.error("A suspiciously small or zero classes or properties found in latest version of " + loader.getOntologyName() + ": Won't index!"); message = "Failed to load - last update had no classes or properties so was rejected"; document.setStatus(Status.FAILED); document.setMessage(message); @@ -77,16 +87,17 @@ public void indexOntologyDocument(OntologyDocument document) throws IndexingExce throw new IndexingException("Empty ontology found", new RuntimeException()); } - } catch (Exception e) { - message = e.getMessage() + ":" + e.getCause().getMessage(); - getLog().error(message); + } catch (Throwable t) { + message = t.getMessage(); + logger.error(message, t); document.setStatus(Status.FAILED); document.setMessage(message); ontologyRepositoryService.update(document); // just set document to failed and return - return; + return result; } - + ResourceUsage.logUsage(logger, "#### Monitoring ",document.getOntologyId() + + ":After loading ontology, before indexing ontology", ":"); document.setStatus(Status.LOADING); ontologyRepositoryService.update(document); // if we get to here, we should have at least loaded the ontology @@ -95,8 +106,12 @@ public void indexOntologyDocument(OntologyDocument document) throws IndexingExce // get all the available indexers for (OntologyIndexer indexer : indexers) { // create the new index - indexer.dropIndex(loader); + indexer.dropIndex(loader.getOntologyName()); + ResourceUsage.logUsage(logger, "#### Monitoring ",document.getOntologyId() + + ":After dropping index" + indexer.toString(), ":"); indexer.createIndex(loader); + ResourceUsage.logUsage(logger, "#### Monitoring ", document.getOntologyId() + + ":After after creating index for " + indexer.toString(), ":"); } // update any ontology meta data @@ -134,21 +149,17 @@ public void indexOntologyDocument(OntologyDocument document) throws IndexingExce if (loader.getVersionNumber() != null) { config.setVersion(loader.getVersionNumber()); } - else { - config.setVersion(new SimpleDateFormat("dd-MM-yyyy").format(new Date())); - } document.setConfig(config); document.setNumberOfTerms(classes.size()); document.setNumberOfProperties(properties.size()); document.setNumberOfIndividuals(individuals.size()); status = Status.LOADED; document.setLoaded(new Date()); - - } catch (Exception e) { - getLog().error("Error indexing " + document.getOntologyId(), e.getMessage()); + result = true; + } catch (Throwable t) { + logger.error("Error indexing " + document.getOntologyId(), t); status = Status.FAILED; - message = e.getMessage(); - throw e; + message = t.getMessage(); } finally { @@ -156,6 +167,35 @@ public void indexOntologyDocument(OntologyDocument document) throws IndexingExce document.setUpdated(new Date()); document.setMessage(message); ontologyRepositoryService.update(document); + return result; + } + } + + @Override + public void removeOntologyDocumentFromIndex(OntologyDocument document) throws IndexingException { + String message = ""; + Status status = Status.FAILED; + + try { + + // get all the available indexers + for (OntologyIndexer indexer : indexers) { + // delete the ontology + indexer.dropIndex(document.getOntologyId()); + } + status = Status.REMOVED; + + } catch (Throwable t) { + logger.error("Error removing index for " + document.getOntologyId(), t.getMessage()); + status = Status.FAILED; + message = t.getMessage(); + throw t; + } + finally { + document.setStatus(status); + document.setUpdated(new Date()); + document.setMessage(message); + ontologyRepositoryService.update(document); } } } diff --git a/ols-mongo/src/main/java/uk/ac/ebi/spot/ols/service/MongoOntologyRepositoryService.java b/ols-mongo/src/main/java/uk/ac/ebi/spot/ols/service/MongoOntologyRepositoryService.java index 58d2e452..60a4458e 100644 --- a/ols-mongo/src/main/java/uk/ac/ebi/spot/ols/service/MongoOntologyRepositoryService.java +++ b/ols-mongo/src/main/java/uk/ac/ebi/spot/ols/service/MongoOntologyRepositoryService.java @@ -10,6 +10,9 @@ import org.springframework.data.mongodb.core.aggregation.TypedAggregation; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.stereotype.Component; + +import com.mongodb.MongoTimeoutException; + import uk.ac.ebi.spot.ols.exception.OntologyRepositoryException; import uk.ac.ebi.spot.ols.model.OntologyDocument; import uk.ac.ebi.spot.ols.model.Status; diff --git a/ols-neo4j/pom.xml b/ols-neo4j/pom.xml index ba4d8116..15ad71ed 100644 --- a/ols-neo4j/pom.xml +++ b/ols-neo4j/pom.xml @@ -1,11 +1,9 @@ - + uk.ac.ebi.spot ols-parent - 3.1.1 + 3.2.1-SNAPSHOT ../ols-parent/pom.xml 4.0.0 @@ -58,28 +56,11 @@ spring-data-rest-core - - - - - - - - - - - org.springframework.data spring-data-rest-webmvc - - - - - - org.neo4j neo4j @@ -91,25 +72,34 @@ ols-core - + + uk.ac.ebi.spot + ontology-tools + + + + net.sourceforge.owlapi + owlapi-distribution + + + + org.junit.jupiter + junit-jupiter-engine + + + + org.junit.jupiter + junit-jupiter-params + test + + + + org.junit.jupiter + junit-jupiter-api + test + - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/ols-neo4j/src/main/java/uk/ac/ebi/spot/ols/loader/BatchNeo4JIndexer.java b/ols-neo4j/src/main/java/uk/ac/ebi/spot/ols/loader/BatchNeo4JIndexer.java index a8a7d1f1..647825b5 100644 --- a/ols-neo4j/src/main/java/uk/ac/ebi/spot/ols/loader/BatchNeo4JIndexer.java +++ b/ols-neo4j/src/main/java/uk/ac/ebi/spot/ols/loader/BatchNeo4JIndexer.java @@ -1,38 +1,39 @@ package uk.ac.ebi.spot.ols.loader; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.json.PackageVersion; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; -import org.neo4j.graphdb.*; +import static uk.ac.ebi.spot.ols.loader.Neo4JIndexerConstants.*; +import static uk.ac.ebi.spot.ols.config.OntologyDefaults.THING; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.neo4j.graphdb.DynamicLabel; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Label; +import org.neo4j.graphdb.Result; +import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.factory.GraphDatabaseFactory; import org.neo4j.graphdb.index.IndexHits; import org.neo4j.graphdb.schema.IndexDefinition; import org.neo4j.graphdb.schema.Schema; import org.neo4j.helpers.collection.MapUtil; import org.neo4j.index.lucene.unsafe.batchinsert.LuceneBatchInserterIndexProvider; -import org.neo4j.kernel.DefaultFileSystemAbstraction; import org.neo4j.unsafe.batchinsert.BatchInserter; import org.neo4j.unsafe.batchinsert.BatchInserterIndex; import org.neo4j.unsafe.batchinsert.BatchInserterIndexProvider; -import org.neo4j.unsafe.batchinsert.BatchInserters; import org.semanticweb.owlapi.model.IRI; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.neo4j.core.GraphDatabase; import org.springframework.stereotype.Component; + import uk.ac.ebi.spot.ols.config.OlsNeo4jConfiguration; import uk.ac.ebi.spot.ols.exception.IndexingException; -import uk.ac.ebi.spot.ols.exception.OntologyLoadingException; import uk.ac.ebi.spot.ols.model.OntologyIndexer; -import uk.ac.ebi.spot.ols.util.OBODefinitionCitation; -import uk.ac.ebi.spot.ols.util.OBOSynonym; -import uk.ac.ebi.spot.ols.util.OBOXref; - -import java.io.File; -import java.util.*; -import java.util.concurrent.TimeUnit; /** * @author Simon Jupp @@ -41,37 +42,18 @@ */ @Component public class BatchNeo4JIndexer implements OntologyIndexer { - private Logger log = LoggerFactory.getLogger(getClass()); - private BatchInserterIndex entities; + private Logger logger = LoggerFactory.getLogger(getClass()); + private BatchInserter inserter; + private BatchInserterIndex index; private BatchInserterIndexProvider indexProvider; - public Logger getLog() { - return log; + public Logger getLogger() { + return logger; } @Autowired private GraphDatabaseService db; - private static Label obsoleteLabel = DynamicLabel.label("Obsolete"); - // this represents a unique term - private static Label mergedClassLabel = DynamicLabel.label("Resource"); - // define a node label for ontology terms - private static Label nodeLabel = DynamicLabel.label("Class"); - private static Label relationLabel = DynamicLabel.label("Property"); - private static Label instanceLabel = DynamicLabel.label("Individual"); - private static Label rootLabel = DynamicLabel.label("Root"); - private static Label _nodeLabel = DynamicLabel.label("_Class"); - private static Label _relationLabel = DynamicLabel.label("_Property"); - private static Label _instanceLabel = DynamicLabel.label("_Individual"); - - // create relationship types - private static RelationshipType refersTo = DynamicRelationshipType.withName("REFERSTO"); - private static RelationshipType isa = DynamicRelationshipType.withName("SUBCLASSOF"); - private static RelationshipType subpropertyof = DynamicRelationshipType.withName("SUBPROPERTYOF"); - private static RelationshipType typeOf = DynamicRelationshipType.withName("INSTANCEOF"); - private static RelationshipType related = DynamicRelationshipType.withName("Related"); - private static RelationshipType treeRelation = DynamicRelationshipType.withName("RelatedTree"); - private Map isaProperties = new HashMap<>(); private Map subPropertyProperties = new HashMap<>(); private Map rdfTypeProperties = new HashMap<>(); @@ -89,171 +71,41 @@ public BatchNeo4JIndexer() { } - private Long getOrCreateNode(BatchInserter inserter, Map nodeMap, OntologyLoader loader, IRI classIri, Label ... nodeLabel) { - - if (!nodeMap.containsKey(classIri.toString())) { - Map properties = new HashMap<>(); - properties.put("olsId", loader.getOntologyName().toLowerCase() + ":" + classIri.toString()); - properties.put("iri", classIri.toString()); - if (classIri.toString().equals("http://www.w3.org/2002/07/owl#Thing")) { - properties.put("label", "Thing"); - properties.put("has_children", true ); - - } - else if (classIri.toString().equals("http://www.w3.org/2002/07/owl#TopObjectProperty")) { - properties.put("label", "TopObjectProperty"); - properties.put("has_children", true ); - } - else { - if (!loader.getTermLabels().containsKey(classIri)) { - properties.put("label", loader.getShortForm(classIri)); - } else { - properties.put("label", loader.getTermLabels().get(classIri)); - } - } - - properties.put("ontology_name", loader.getOntologyName()); - properties.put("ontology_prefix", loader.getPreferredPrefix()); - properties.put("ontology_iri", loader.getOntologyIRI().toString()); - properties.put("is_obsolete", loader.isObsoleteTerm(classIri)); - properties.put("is_defining_ontology", loader.isLocalTerm(classIri)); - // && loader.getRelatedChildTerms(classIri).isEmpty() - - // if it's an individual, it can't have children, but it may be punned URI to a class to we need to check - List