diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2349600 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM openjdk:13 +MAINTAINER Mike Travers "mt@alum.mit.edu" + +ENV PORT 1996 +EXPOSE 1996 + +ADD target ~/target +WORKDIR ~/target + +ENTRYPOINT ["java", "-jar", "enflame-standalone.jar", "opencandel-config.edn"] diff --git a/README.md b/README.md index 71e2090..51ea3b0 100644 --- a/README.md +++ b/README.md @@ -24,11 +24,11 @@ See [sample config](resources/candel-config.edn) ## To run locally from source: -Copy `resources/candel-config.edn` to `deploy/candel-config.edn`, filling out as appropriate. +Create the file `deploy/launch-config.edn` according to your needs (see the sample configs in `resources`). Then: lein launch -This will compile the front-end, lauch a server, and open a browser windoe. +This will compile the front-end, lauch the back-end, and open a browser windoe. ## Documentation generation diff --git a/doc/build-guide.sh b/bin/build-guide.sh similarity index 100% rename from doc/build-guide.sh rename to bin/build-guide.sh diff --git a/bin/build.sh b/bin/build.sh new file mode 100755 index 0000000..f988d34 --- /dev/null +++ b/bin/build.sh @@ -0,0 +1,39 @@ +# Build Docker image + +# ID of AWS Elastic Container Registry +export ECR=733151965047.dkr.ecr.us-east-1.amazonaws.com + +# TODO parameterize properly + +export VERSION=$(git rev-parse --short HEAD) +echo Building version $VERSION + +bin/build-guide.sh + +# get latest version of schema (TODO: is this a good idea?) +# doesn't seem to actually work +# git submodule update --init --recursive + +# TODO needs rethinking +# Run Alzabo to build schemas +# cd alzabo; lein with-profile prod do clean, run documentation candel "*", cljsbuild once; cd .. + +lein do clean, uberjar + +# TODO abort if lein fails, duh. + +# Build Docker image + +docker build -t cbio . + +# verify +# docker run -p 8080 cbio + + +docker tag cbio:latest $ECR/cbio:latest + +# Upload to AWS repository + +aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $ECR + +docker push $ECR/cbio:latest diff --git a/deploy/README-devops.org b/deploy/README-devops.org new file mode 100644 index 0000000..16de048 --- /dev/null +++ b/deploy/README-devops.org @@ -0,0 +1,65 @@ +* Scope of this document + +This describes the simplified CANDEL/Enflame architecture used for the public CandelBIO website. + +* Deployment + +Once App Runner is set up, pushing a new Docker image will reboot the server. + +[[bin/build.sh]] does all the necessary steps + +** Preconditions +- Docker must be running +- AWS cred must be in place +- emacs available from shell + + +* AWS Services + +These were all cobbled together by hand, I suppose the TODO is to make a CloudFormation them. See [[deploy/resources]] for more details. + +** Datomic Cloud + +Launched from AWS Console: https://docs.datomic.com/cloud/getting-started/start-system.html + +This resuls in creating three CloudFormation stacks, one master and two subordinates. + +Current master is https://us-east-1.console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/stackinfo?filteringText=&filteringStatus=active&viewNested=true&stackId=arn%3Aaws%3Acloudformation%3Aus-east-1%3A733151965047%3Astack%2FPublicCANDEL5%2F0eee3a40-b863-11ed-9e49-1270ef971d67 + +You need to copy the ClientApiGatewayEnpoint output to Enflame's configuration. The current value is https://nazpex6ueb.execute-api.us-east-1.amazonaws.com + +** ECR (Container Registry) + + +This holds the Docker container that App Runner uses. + +Current instance: https://us-east-1.console.aws.amazon.com/ecr/repositories/private/733151965047/cbio?region=us-east-1 + +The name of the most recent image is: 733151965047.dkr.ecr.us-east-1.amazonaws.com/cbio:latest +This needs to be supplied to the build.sh script + +** IAM + +Service role for App Runner + +https://us-east-1.console.aws.amazon.com/iamv2/home?region=us-east-1#/roles/details/Enflame-OpenCANDEL?section=permissions + +** App Runner + +App Runner service has to be configured with the + + +https://us-east-1.console.aws.amazon.com/apprunner/home?region=us-east-1#/services/dashboard?service_arn=arn%3Aaws%3Aapprunner%3Aus-east-1%3A733151965047%3Aservice%2FEnflame-OpenCANDEL%2F68f5614bbe7248d2896d397b1e3b2033&active_tab=logs + +Service: Enflame-OpenCANDEL +Public endpint: https://7gbxx4vxvn.us-east-1.awsapprunner.com/ + +** DynamoDB + +For library storage + +https://us-east-1.console.aws.amazon.com/dynamodbv2/home?region=us-east-1#table?name=OpenCANDEL_EnflameLibrary + + + +** TODO storage service for library diff --git a/deploy/candel-config.edn b/deploy/candel-config.edn new file mode 100644 index 0000000..e6221af --- /dev/null +++ b/deploy/candel-config.edn @@ -0,0 +1,18 @@ +;;; TODO Use aero to default to resources/candel-config.edn +{:source {:type :candel + :candelabra-endpoint "https://pici-prod-v1.candel.parkerici.org/" + :insecure-https false} + :schema "resources/candel-schema-1-3-1.edn" + :query-generator :candel-generate + :library {:gcs-project "pici-dev" + :gcs-console-url "https://console.cloud.google.com/datastore/entities;kind=EnflameItem;ns=__$DEFAULT$__;sortCol=date-created;sortDir=DESCENDING/query/kind?project=pici-dev"} + :rh-cards [:candel/db + :query + :candel/wick + :share + :compact ;debug only + :browser + ] + :dev? true ;TODO control this somehow maybe aero/env + :port 1983 + } diff --git a/deploy/resources b/deploy/resources new file mode 100644 index 0000000..03f4612 --- /dev/null +++ b/deploy/resources @@ -0,0 +1,180 @@ +Note: this is a dump of relevant AWS resources (results trimmed to only include the OpenCANDEL relevax`nt ones) + + +14:49:59 ~/os/enflame (opencandel) ⪢ aws apprunner list-services +{ + "ServiceSummaryList": [ + { + "ServiceName": "Enflame-OpenCANDEL", + "ServiceId": "68f5614bbe7248d2896d397b1e3b2033", + "ServiceArn": "arn:aws:apprunner:us-east-1:733151965047:service/Enflame-OpenCANDEL/68f5614bbe7248d2896d397b1e3b2033", + "ServiceUrl": "7gbxx4vxvn.us-east-1.awsapprunner.com", + "CreatedAt": "2023-03-01T16:01:27-08:00", + "UpdatedAt": "2023-03-01T16:01:27-08:00", + "Status": "RUNNING" + } + ] +} + +15:03:07 ~/os/enflame (opencandel) ⪢ aws iam list-roles +{ + "Roles": [ + { + "Path": "/", + "RoleName": "Enflame-OpenCANDEL", + "RoleId": "AROA2VM2DF53ZHMLN5XGO", + "Arn": "arn:aws:iam::733151965047:role/Enflame-OpenCANDEL", + "CreateDate": "2023-03-02T00:58:08+00:00", + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + }, + "Action": "sts:AssumeRole" + }, + { + "Effect": "Allow", + "Principal": { + "Service": "tasks.apprunner.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] + }, + "Description": "Role for Enflame OpenCANDEL server", + "MaxSessionDuration": 3600 + } + ] +} + + + +18:23:09 ~/os/enflame (opencandel) ⪢ aws iam list-role-policies --role-name "Enflame-OpenCANDEL" +{ + "PolicyNames": [ + "OpenCANDEL_Enflame_Library" + ] +} + + + +18:22:49 ~/os/enflame (opencandel) ⪢ aws iam get-role-policy --role-name "Enflame-OpenCANDEL" --policy-name "OpenCANDEL_Enflame_Library" +{ + "RoleName": "Enflame-OpenCANDEL", + "PolicyName": "OpenCANDEL_Enflame_Library", + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "dynamodb:PutItem", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:Query", + "dynamodb:UpdateItem" + ], + "Resource": "arn:aws:dynamodb:us-east-1:733151965047:table/OpenCANDEL_EnflameLibrary" + } + ] + } +} + + + +14:52:19 ~/os/enflame (opencandel) ⪢ aws ecr describe-repositories +{ + "repositories": [ + + { + "repositoryArn": "arn:aws:ecr:us-east-1:733151965047:repository/cbio", + "registryId": "733151965047", + "repositoryName": "cbio", + "repositoryUri": "733151965047.dkr.ecr.us-east-1.amazonaws.com/cbio", + "createdAt": "2023-03-01T13:32:19-08:00", + "imageTagMutability": "MUTABLE", + "imageScanningConfiguration": { + "scanOnPush": false + }, + "encryptionConfiguration": { + "encryptionType": "AES256" + } + } + ] +} + + +14:50:11 ~/os/enflame (opencandel) ⪢ aws cloudformation list-stacks +{ + "StackSummaries": [ + { + "StackId": "arn:aws:cloudformation:us-east-1:733151965047:stack/PublicCANDEL5-Compute-93S9RT2L5TUI/bbcf46f0-b863-11ed-932d-0ab2700ca4cf", + "StackName": "PublicCANDEL5-Compute-93S9RT2L5TUI", + "TemplateDescription": "Creates compute resources needed to run Datomic.", + "CreationTime": "2023-03-01T19:03:21.404000+00:00", + "StackStatus": "CREATE_COMPLETE", + "ParentId": "arn:aws:cloudformation:us-east-1:733151965047:stack/PublicCANDEL5/0eee3a40-b863-11ed-9e49-1270ef971d67", + "RootId": "arn:aws:cloudformation:us-east-1:733151965047:stack/PublicCANDEL5/0eee3a40-b863-11ed-9e49-1270ef971d67", + "DriftInformation": { + "StackDriftStatus": "NOT_CHECKED" + } + }, + { + "StackId": "arn:aws:cloudformation:us-east-1:733151965047:stack/PublicCANDEL5-StorageF7F305E7-UD47ELG33U9Z/134d9cc0-b863-11ed-b08e-128bb3d0467b", + "StackName": "PublicCANDEL5-StorageF7F305E7-UD47ELG33U9Z", + "TemplateDescription": "Creates storage resources needed to run Datomic.", + "CreationTime": "2023-03-01T18:58:38.643000+00:00", + "StackStatus": "CREATE_COMPLETE", + "ParentId": "arn:aws:cloudformation:us-east-1:733151965047:stack/PublicCANDEL5/0eee3a40-b863-11ed-9e49-1270ef971d67", + "RootId": "arn:aws:cloudformation:us-east-1:733151965047:stack/PublicCANDEL5/0eee3a40-b863-11ed-9e49-1270ef971d67", + "DriftInformation": { + "StackDriftStatus": "NOT_CHECKED" + } + }, + { + "StackId": "arn:aws:cloudformation:us-east-1:733151965047:stack/PublicCANDEL5/0eee3a40-b863-11ed-9e49-1270ef971d67", + "StackName": "PublicCANDEL5", + "TemplateDescription": "Creates resources needed to run Datomic. The stack name is the name of the Datomic system.", + "CreationTime": "2023-03-01T18:58:31.189000+00:00", + "StackStatus": "CREATE_COMPLETE", + "DriftInformation": { + "StackDriftStatus": "NOT_CHECKED" + } + }, + + ] +} + +20:25:31 ~/os/enflame (opencandel) ⪢ aws dynamodb describe-table --table-name "OpenCANDEL_EnflameLibrary" +{ + "Table": { + "AttributeDefinitions": [ + { + "AttributeName": "entityId", + "AttributeType": "S" + } + ], + "TableName": "OpenCANDEL_EnflameLibrary", + "KeySchema": [ + { + "AttributeName": "entityId", + "KeyType": "HASH" + } + ], + "TableStatus": "ACTIVE", + "CreationDateTime": "2023-03-09T09:57:35.493000-08:00", + "ProvisionedThroughput": { + "NumberOfDecreasesToday": 0, + "ReadCapacityUnits": 1, + "WriteCapacityUnits": 1 + }, + "TableSizeBytes": 57488, + "ItemCount": 3, + "TableArn": "arn:aws:dynamodb:us-east-1:733151965047:table/OpenCANDEL_EnflameLibrary", + "TableId": "f29723c1-b287-48da-8c8b-6739caaed63e" + } +} diff --git a/doc/tutorial.org b/doc/tutorial.org index 0e35eb5..d1e6359 100644 --- a/doc/tutorial.org +++ b/doc/tutorial.org @@ -17,7 +17,7 @@ When building a query, you select blocks that represent the type of object you are interested in, and other blocks that represent attribiutes of those objects, or relationships to other objects. To start with a simple example, here' a query that will find all the =subject= objects with race =african-american=: #+BEGIN_EXPORT html - + #+END_EXPORT Click the [[file:Screen_Shot_2021-05-14_at_12.04.21_PM.png]] button to run the query, and observe the results. @@ -33,7 +33,7 @@ Next, try changing the query. Here are some things you can try: Here's a slightly more complex query. Again we are searching for subjects, but now we have multiple constraints on the attributes. The query starts off with two constraints...try running it. #+BEGIN_EXPORT html - + #+END_EXPORT Note that there is a third, unconnected attribute block [[file:Screen_Shot_2021-05-15_at_11.57.09_AM.png]] lying around. It doesn't have any effect until you add it to the query – try doing that by dragging in under the ... block and running the query again. @@ -51,7 +51,7 @@ TODO example has position problems # [subjects where [sample is [samples]] and [race is white]] #+BEGIN_EXPORT html - + #+END_EXPORT Now try constraining the subquery by dragging the [[file:Screen_Shot_2021-05-12_at_4.55.29_PM.png]] block into the sample query block and running the query again. @@ -64,7 +64,7 @@ TODO this should be output everything probably, # [variants where gene is EGFR] #+BEGIN_EXPORT html - + #+END_EXPORT You can try substituting a different gene name. @@ -76,7 +76,7 @@ This example shows how you can produce row-specific counts. This query finds all # Subjects with disease (counts #+BEGIN_EXPORT html - + #+END_EXPORT Try adding the [[file:Screen_Shot_2021-05-17_at_9.41.13_AM.png]] block to constrain diseases based on a substring of the disease name. @@ -88,7 +88,7 @@ Here's a more complex example. This query returns subjects that have a variant i # subjects where variant is variants where gene is EGFR #+BEGIN_EXPORT html - + #+END_EXPORT Try changing the gene, or experimenting with the outputs. @@ -99,7 +99,7 @@ Try changing the gene, or experimenting with the outputs. Enflame can not only generate queries, but generate data visualizations of the results. Here's an example which summarizes the counts of clinical observations. #+BEGIN_EXPORT html - + #+END_EXPORT diff --git a/project.clj b/project.clj index ba0fc8f..51627a9 100644 --- a/project.clj +++ b/project.clj @@ -1,27 +1,33 @@ (defproject enflame "0.0.29-SNAPSHOT" - :dependencies [[org.clojure/clojure "1.10.1"] - [clj-http "3.12.1" + :dependencies [[org.clojure/clojure "1.11.1"] + [clj-http "3.12.3" :exclusions [commons-codec]] - [cheshire "5.10.0"] + [cheshire "5.11.0"] + + [com.datomic/client-cloud "1.0.122"] ;; Ring and its large family - [org.eclipse.jetty/jetty-client "9.4.12.v20180830"] ;has to match ring version of jetty - [org.eclipse.jetty/jetty-server "9.4.12.v20180830"] - [org.eclipse.jetty/jetty-http "9.4.12.v20180830"] - [org.eclipse.jetty/jetty-util "9.4.12.v20180830"] - - [ring "1.8.0"] - [ring/ring-jetty-adapter "1.7.1"] - [ring/ring-defaults "0.3.2"] - [ring-logger "1.0.1"] - [ring-oauth2 "0.1.4"] + [org.eclipse.jetty/jetty-client "9.4.48.v20220622"] ;has to match ring version of jetty + [org.eclipse.jetty/jetty-server "9.4.48.v20220622"] + [org.eclipse.jetty/jetty-http "9.4.48.v20220622"] + [org.eclipse.jetty/jetty-util "9.4.48.v20220622"] + + [ring "1.9.6"] + [ring/ring-jetty-adapter "1.9.6"] + [ring/ring-defaults "0.3.4"] + [ring-logger "1.1.1"] + [ring-oauth2 "0.2.0" :exclusions [org.apache.httpcomponents/httpcore]] [org.slf4j/slf4j-simple "1.7.26"] ;required to turn off warning - [com.taoensso/timbre "4.10.0"] + [io.aviso/pretty "1.3"] + [com.taoensso/timbre "4.10.0" + :exclusions [io.aviso/pretty]] [org.clojure/data.csv "0.1.4"] [compojure "1.6.1" :exclusions [ring.core ring.codec]] - [ring-middleware-format "0.7.4" :exclusions [javax.xml.bind/jaxb-api]] + [ring-middleware-format "0.7.5" :exclusions [javax.xml.bind/jaxb-api]] [bk/ring-gzip "0.3.0"] [trptcolin/versioneer "0.2.0"] + + ;; Note: the gcs stuff is not going to used post PICI and could be removed. So annoying that it is hard to have different configs. [com.google.cloud/google-cloud-datastore "1.105.7" ;; TODO have Cognitect or someone review this :exclusions [com.google.errorprone/error_prone_annotations @@ -29,27 +35,37 @@ org.apache.httpcomponents/httpclient com.google.guava/guava com.fasterxml.jackson.core/jackson-core]] + + [com.cognitect.aws/api "0.8.656"] + [com.cognitect.aws/endpoints "1.1.12.415"] + [com.cognitect.aws/dynamodb "825.2.1262.0"] + [environ "1.1.0"] [me.raynes/fs "1.4.6"] - [org.parkerici/multitool "0.0.19"] + [org.parkerici/multitool "0.0.26"] [com.cemerick/url "0.1.1"] [org.clojure/data.xml "0.2.0-alpha6"] [org.clojure/clojurescript "1.10.520"] [aristotle/aristotle "0.1.0" - :exclusions [org.apache.jena/apache-jena-libs]] ;asking for trouble + :exclusions [org.apache.jena/apache-jena-libs ;asking for trouble + ;; TODO this fixed a problem with longs, but caused other issues... + ; javax.xml.bind/jaxb-api + ]] [org.apache.jena/apache-jena-libs "3.16.0" :extension "pom"] [metasoarous/oz "1.6.0-alpha6" ; warning: later versions seem to have broken dependencies :exclusions [cljsjs/vega ; we insert a later version of Vega to fix some bugs cljsjs/vega-lite cljsjs/vega-embed - cljsjs/vega-tooltip]] + cljsjs/vega-tooltip + com.taoensso/encore]] + [cljsjs/vega "5.20.2-0"] [cljsjs/vega-lite "5.1.1-0"] [cljsjs/vega-embed "6.19.0-0"] [cljsjs/vega-tooltip "0.27.0-0"] - [org.parkerici/blockoid "0.3.6"] + [org.parkerici/blockoid "0.3.10"] [reagent "0.8.1"] [re-frame "0.10.9" :exclusions [org.clojure/clojurescript]] @@ -69,12 +85,10 @@ :min-lein-version "2.5.3" :main ^:skip-aot org.parkerici.enflame.core - ;; This builds and launches the CANDEL version - ;;; note: you have to fill out the config file in resources/candel-config.edn and put it in deploy folder :aliases {"launch" ["do" "clean" ["cljsbuild" "once" "prod"] - ["run" "deploy/candel-config.edn"]]} + ["run" "deploy/launch-config.edn"]]} :source-paths ["src/clj" "src/cljs" "src/cljc"] :test-paths ["test/clj" "test/cljs" "test/cljc"] @@ -85,6 +99,7 @@ :css-dirs ["resources/public/css"]} :uberjar-name "enflame-standalone.jar" + :resource-paths ["resources"] :profiles {:dev @@ -128,7 +143,8 @@ :compiler {:main org.parkerici.enflame.core :output-to "resources/public/js/compiled/app.js" :output-dir "resources/public/js/compiled/outprod" - :optimizations :simple - :infer-externs true + :asset-path "js/compiled/outprod" + :optimizations :none ; was :simple but that stopped working for unknown reasons + :infer-externs true :closure-defines {goog.DEBUG false} :pretty-print false}}]}) diff --git a/resources/opencandel-config.edn b/resources/opencandel-config.edn new file mode 100644 index 0000000..823f86d --- /dev/null +++ b/resources/opencandel-config.edn @@ -0,0 +1,28 @@ +{:source {:type :datomic-cloud ;The data source. + :config {:server-type :ion ;Passed directly to Datomic Client API + :region "us-east-1" ;; e.g. us-east-1 + :system "PublicCANDEL5" + ;; :creds-profile "" + :endpoint "https://nazpex6ueb.execute-api.us-east-1.amazonaws.com"} + } + + :schema "candel-schema-1-3-1.edn" ;filename of an Alzabo schema + + :library {:type :dynamodb + :table "OpenCANDEL_EnflameLibrary" + :console-url "https://us-east-1.console.aws.amazon.com/dynamodbv2/home?region=us-east-1#table?name=OpenCANDEL_EnflameLibrary" + } + + :query-generator :candel-generate ;The query generator + + :rh-cards [:candel/db ;Sequence of cards to show in right-hand side panel + :query + ;; I guess no point to this until Wick server is brought up + ;; :candel/wick + :share + ;; :compact ;debug only + :browser + ] + :port 1996 ;Port on which to serve application + :dev? true ;debug only + } diff --git a/resources/public/doc b/resources/public/doc new file mode 120000 index 0000000..7e57b0f --- /dev/null +++ b/resources/public/doc @@ -0,0 +1 @@ +../../doc \ No newline at end of file diff --git a/resources/public/index.html b/resources/public/index.html index 0fcc0cb..e861979 100644 --- a/resources/public/index.html +++ b/resources/public/index.html @@ -3,7 +3,7 @@ Enflame query builder - + diff --git a/resources/public/schema b/resources/public/schema new file mode 120000 index 0000000..fa2ecdd --- /dev/null +++ b/resources/public/schema @@ -0,0 +1 @@ +../../../alzabo/resources/public/schema/1.3.1 \ No newline at end of file diff --git a/src/clj/org/parkerici/enflame/admin.clj b/src/clj/org/parkerici/enflame/admin.clj index 766a693..8aff365 100644 --- a/src/clj/org/parkerici/enflame/admin.clj +++ b/src/clj/org/parkerici/enflame/admin.clj @@ -3,6 +3,7 @@ [org.parkerici.enflame.html :as html] [hiccup.util :as hu] [clojure.java.shell :as sh] + [org.parkerici.multitool.core :as u] )) (defn map-table @@ -17,8 +18,9 @@ (hu/escape-html (str (get map key))))]])]]) (defn git-info [] - {:commit (:out (sh/sh "git" "log" "-1" "--format=short")) - :branch (:out (sh/sh "git" "rev-parse" "--abbrev-ref" "HEAD"))}) + (u/ignore-errors + {:commit (:out (sh/sh "git" "log" "-1" "--format=short")) + :branch (:out (sh/sh "git" "rev-parse" "--abbrev-ref" "HEAD"))})) (defn view [req] diff --git a/src/clj/org/parkerici/enflame/config.clj b/src/clj/org/parkerici/enflame/config.clj index c02e421..f736e2a 100644 --- a/src/clj/org/parkerici/enflame/config.clj +++ b/src/clj/org/parkerici/enflame/config.clj @@ -3,30 +3,37 @@ [clojure.edn :as edn] [clojure.pprint :as pprint] [org.parkerici.multitool.core :as u] + [org.parkerici.enflame.schema :as schema] + [clojure.java.io :as io] )) ;;; TODO Aero-ize (def the-config (atom nil)) +(defn config + [& keys] + (assert @the-config "Config not set") + (get-in @the-config keys)) + +;;; Called from server +;;; Here because schema is cljc +(defn read-schema [from] + (-> from + io/resource + slurp + edn/read-string)) + (defn load-config [file] (let [env-config {} ; if we want to supply some config from environment - file-config (edn/read-string (slurp file)) + file-config (edn/read-string (slurp (io/resource file))) config (merge env-config file-config)] (pprint/pprint config) - (reset! the-config config))) + (reset! the-config config) + (schema/set-schema (read-schema (config :schema))))) -(defn config - ([& keys] (get-in @the-config keys)) - ([] @the-config )) -;;; Called from server -;;; TODO use version -(u/defn-memoized read-schema [version] - (-> (config :schema) - slurp - edn/read-string)) ;;; TODO diff --git a/src/clj/org/parkerici/enflame/datomic_client.clj b/src/clj/org/parkerici/enflame/datomic_client.clj new file mode 100644 index 0000000..ac08fbb --- /dev/null +++ b/src/clj/org/parkerici/enflame/datomic_client.clj @@ -0,0 +1,46 @@ +(ns org.parkerici.enflame.datomic-client + (:require [datomic.client.api :as d] + [taoensso.timbre :as log] + [org.parkerici.multitool.core :as u] + [org.parkerici.enflame.config :as config] + )) + +;;; TODO do these at a reasonable time + +(u/def-lazy client (d/client (config/config :source :config))) + +;;; OK probably we want to handle multiple databases +(u/defn-memoized conn + [db] + (d/connect @client {:db-name db})) + + + +(defn dbs + [] + (d/list-databases @client {})) + +;;; Query optimizer +;;; Cute hack, and no idea why Datomic doesn't just do this + +(u/defn-memoized db-stats + [db] + (d/db-stats (d/db (conn db)))) + +(defn db-att-count + [db att] + (get-in (db-stats db) [:attrs att :count] 0)) + +;;; Resorts where clauses so smaller searches come first +(defn optimize-query + [db query] + (update query :where + (fn [clauses] + (sort-by (comp (partial db-att-count db) second) + clauses)))) + + +(defn query + [db query args] + (d/q {:query (optimize-query db query) + :args (cons (d/db (conn db)) args)})) diff --git a/src/clj/org/parkerici/enflame/embed_server.clj b/src/clj/org/parkerici/enflame/embed_server.clj index e3ad4ba..2f470c5 100644 --- a/src/clj/org/parkerici/enflame/embed_server.clj +++ b/src/clj/org/parkerici/enflame/embed_server.clj @@ -31,7 +31,7 @@ [:div.scrollbar.scrollbar-primary {:style {:height "300px" :width "100%" :overflow-y "scroll"}} [:div#results.force-overflow]]]] [:script {:src "js/compiled/app.js"}]] - [:script "org.parkerici.enflame.embed.init();"] + [:script "org.parkerici.enflame.core.embed();"] ])) "text/html")) diff --git a/src/clj/org/parkerici/enflame/gcs.clj b/src/clj/org/parkerici/enflame/gcs.clj index 1855577..e272ad1 100644 --- a/src/clj/org/parkerici/enflame/gcs.clj +++ b/src/clj/org/parkerici/enflame/gcs.clj @@ -8,25 +8,26 @@ ;;; Interface to GCS Datastore service -(def service +;;; TODO this is very bad. A little less bad made lazy +(u/def-lazy service (.getService (-> (DatastoreOptions/newBuilder) (.setProjectId (config/config :library :gcs-project)) (.build)))) (u/defn-memoized key-factory [kind] - (-> (.newKeyFactory service) + (-> (.newKeyFactory @service) (.setKind kind))) (defn get-entity-key [kind ent-id] - (-> (.newKeyFactory service) + (-> (.newKeyFactory @service) (.setKind kind) (.newKey ent-id))) (defn delete-entity [ent-id] - (.delete service (into-array [(get-entity-key ent-id)]))) + (.delete @service (into-array [(get-entity-key ent-id)]))) (defn to-map [ent] @@ -40,7 +41,7 @@ [kind ent-id] (let [k (get-entity-key kind ent-id)] (when k - (to-map (.get service k))))) + (to-map (.get @service k))))) ;;: TODO parameterize query maybe (defn list-items @@ -49,7 +50,7 @@ (.setKind kind))] (map to-map (iterator-seq - (.run service (.build query)))))) + (.run @service (.build query)))))) (defn latest-item [kind field] @@ -60,7 +61,7 @@ (to-map (first (iterator-seq - (.run service (.build query))))))) + (.run @service (.build query))))))) (defn all-items [kind] @@ -69,7 +70,7 @@ )] (map to-map (iterator-seq - (.run service (.build query)))))) + (.run @service (.build query)))))) ;;; Not presently called (defn upload-property-names @@ -77,7 +78,7 @@ (let [key-factory (key-factory "PropertyName") v (map name (keys fields)) entities (map #(.build (Entity/newBuilder (.newKey key-factory %))) v)] - (.put service (into-array entities)))) + (.put @service (into-array entities)))) (defn big-string [s] (-> s @@ -98,10 +99,10 @@ (defn upload [kind item big-keys] (let [key (->> (.newKey (key-factory kind)) - (.allocateId service)) + (.allocateId @service)) builder (-> (Entity/newBuilder key) (add-fields-to-entity item big-keys))] - (.add service (.build builder)))) + (.add @service (.build builder)))) ;;; Datastore does not support OR or IN operatore, so we fake it ;;; Warning: this does a crossproduct on all multiple-valued prop filters, can be expensive @@ -117,4 +118,4 @@ (when property-filters (.setFilter query property-filters)) (into [] (iterator-seq - (.run service (.build query)))))) + (.run @service (.build query)))))) diff --git a/src/clj/org/parkerici/enflame/library/core.clj b/src/clj/org/parkerici/enflame/library/core.clj new file mode 100644 index 0000000..0e81761 --- /dev/null +++ b/src/clj/org/parkerici/enflame/library/core.clj @@ -0,0 +1,32 @@ +(ns org.parkerici.enflame.library.core + (:require [org.parkerici.enflame.gcs :as gcs] + [org.parkerici.enflame.library.dynamodb :as dyna] + [org.parkerici.enflame.config :as config] + ) + ) + +;;; GCS implementation + +(def big-keys #{::blockdef ::image}) + +(defn upload + [item] + (case (config/config :library :type) + :gcs (gcs/upload "EnflameItem" item big-keys) + :dynamodb (dyna/upload item) + )) + +(defn get-item + [key] + (case (config/config :library :type) + :gcs (gcs/get-item "EnflameItem" (Long. key)) + :dynamodb (dyna/get-item key) + )) + +(defn list-items + [] + (case (config/config :library :type) + :gcs (gcs/list-items "EnflameItem") + :dynamodb (dyna/list-items) + ) + ) diff --git a/src/clj/org/parkerici/enflame/library/dynamodb.clj b/src/clj/org/parkerici/enflame/library/dynamodb.clj new file mode 100644 index 0000000..3c4ff2a --- /dev/null +++ b/src/clj/org/parkerici/enflame/library/dynamodb.clj @@ -0,0 +1,93 @@ +(ns org.parkerici.enflame.library.dynamodb + (:require [cognitect.aws.client.api :as aws] + [org.parkerici.enflame.library.item :as item] + [org.parkerici.enflame.config :as config] + [org.parkerici.multitool.core :as u] + ) + ) + +(def dyna (aws/client {:api :dynamodb})) + +;;; Invoke with error handling +(defn invoke-with-eh + [arg-map] + (let [resp (aws/invoke dyna arg-map)] + (when (:cognitect.anomalies/category resp) + (throw (ex-info "DynamoDB exception" {:arg-map arg-map :resp resp}))) + resp)) + +;;; When debugging, might want to Turn off the magic +#_ +(defn invoke-with-eh + [arg-map] + (aws/invoke dyna arg-map)) + +(defn table + [] + (config/config :library :table)) + +;;; Delete the table – beware, data loss! +(defn delete-table + [] + (invoke-with-eh + {:op :DeleteTable + :request {:TableName (table)}})) + +(defn create-table + [] + (invoke-with-eh + {:op :CreateTable + :request {:TableName (table) + ;; Note: only key fields should be defined here + :AttributeDefinitions [{:AttributeName "entityId" + :AttributeType "S"} + ] + + :KeySchema [{:AttributeName "entityId" + :KeyType "HASH"} + ] + :ProvisionedThroughput {:ReadCapacityUnits 1 + :WriteCapacityUnits 1}}})) + + +(defn from-item + [item] + (-> (into {} + (map (fn [[k v]] + [(keyword "org.parkerici.enflame.library.item" (name k)) + (or (:N v) (:S v))]) + item)) + (update :org.parkerici.enflame.library.item/date-created + #(-> % parse-long)))) ;view calls java.util.Date. + +(defn list-items + [] + (map from-item + (:Items + (invoke-with-eh {:op :Scan :request {:TableName (table)}})))) + +(defn to-item + [thing] + (let [thang (assoc thing :entityId (str (java.util.UUID/randomUUID))) + ks (keys thang)] + (zipmap ks + (map #(let [v (% thang)] + (cond (number? v) + {:N (str v)} + :else + {:S (str v)})) + ks)))) + +(defn upload + [thing] + (invoke-with-eh + {:op :PutItem + :request {:TableName (table) + :Item (to-item thing)}}) ) + +(defn get-item + [k] + (-> (invoke-with-eh {:op :GetItem :request {:TableName (table) :Key {"entityId" {:S k}}}}) + :Item + from-item)) + diff --git a/src/clj/org/parkerici/enflame/library/view.clj b/src/clj/org/parkerici/enflame/library/view.clj index 9fa25fa..04c7035 100644 --- a/src/clj/org/parkerici/enflame/library/view.clj +++ b/src/clj/org/parkerici/enflame/library/view.clj @@ -1,6 +1,7 @@ (ns org.parkerici.enflame.library.view (:require [org.parkerici.enflame.html :as html] [org.parkerici.enflame.gcs :as gcs] + [org.parkerici.enflame.library.core :as lib] [org.parkerici.enflame.library.item :as item] [org.parkerici.enflame.config :as config] [org.parkerici.multitool.core :as u] @@ -28,15 +29,15 @@ (defn library-items [] - (map item/localize-item (gcs/list-items "EnflameItem"))) + (map item/localize-item (lib/list-items))) (defn view [] (html/html-frame "Library" [:div.container.col-12 - [:a {:href (config/config :library :gcs-console-url)} - "Edit in GCS console"] + [:a {:href (config/config :library :console-url)} + "Edit in cloud console"] [:table.table [:tbody (for [item (reverse (sort-by ::item/date-created (library-items)))] diff --git a/src/clj/org/parkerici/enflame/server.clj b/src/clj/org/parkerici/enflame/server.clj index 8c1463b..16f943f 100644 --- a/src/clj/org/parkerici/enflame/server.clj +++ b/src/clj/org/parkerici/enflame/server.clj @@ -1,12 +1,13 @@ (ns org.parkerici.enflame.server (:require [org.parkerici.enflame.datomic-relay :as datomic] + [org.parkerici.enflame.datomic-client :as datomic-client] [org.parkerici.enflame.download :as download] [org.parkerici.enflame.embed-server :as embed] [org.parkerici.enflame.admin :as admin] [org.parkerici.enflame.schema :as schema] [org.parkerici.enflame.config :as config] [org.parkerici.enflame.oauth :as oauth] - [org.parkerici.enflame.gcs :as gcs] + [org.parkerici.enflame.library.core :as lib] [org.parkerici.enflame.library.item :as item] [org.parkerici.enflame.library.view :as library-view] [org.parkerici.multitool.core :as u] @@ -42,8 +43,8 @@ _query (read-string query) _args (if (u/nullish? args) [] (read-string args)) _limit (if (u/nullish? limit) nil (Integer. limit)) - candelabra-token (get-in req [:cookies "candelabra-token" :value]) - results (datomic/query db _query _args candelabra-token config) + results #_ (datomic/query db _query _args candelabra-token config) + (datomic-client/query db _query _args) clipped (if _limit (take _limit results) results)] (response/response {:count (count results) :clipped (count clipped) :results clipped}))) @@ -61,13 +62,13 @@ (defn handle-save [req] (let [item (get-in req [:params :item]) - response (gcs/upload "EnflameItem" item item/big-keys)] + response (lib/upload item)] (response/response ;response^4 {:response response}))) (defn handle-get [key] - (let [item (item/localize-item (gcs/get-item "EnflameItem" (Long. key)))] + (let [item (item/localize-item (lib/get-item key))] ;; TODO handle not found (response/response item))) @@ -97,11 +98,18 @@ "candelabra-token" user-creds)] respon)))) +;;; Old CANDEL +#_ (defn handle-databases [req config] (let [candelabra-token (get-in req [:cookies "candelabra-token" :value])] (datomic/dbs candelabra-token config))) +;;; Open CANDSL +(defn handle-databases + [req config] + (datomic-client/dbs)) + (defn app-routes [config] (routes @@ -124,8 +132,8 @@ (GET "/config" req (response/response (config/config))) (GET "/databases" req ;TODO candel specific. Fold into schema (response/response (handle-databases req config))) - (GET "/schema" [version] - (response/response (config/read-schema version))) + (GET "/schema" [version] ;TODO version ignored + (response/response @schema/the-schema)) (GET "/query" req (handle-query req config)) (context "/library" [] (GET "/get" [key] @@ -141,7 +149,7 @@ (def site-defaults (-> middleware/site-defaults (assoc-in [:security :anti-forgery] false) ;interfering with save? - (assoc-in [:session :cookie-attrs :same-site] :lax) ;for oauth + #_ (assoc-in [:session :cookie-attrs :same-site] :lax) ;for oauth (assoc-in [:session :store] common-store) (assoc-in [:static :resources] nil))) ;this needs to go after oauth @@ -159,7 +167,7 @@ (when-not (some #(= % (:uri message)) no-log) (log/log level throwable message)))}) (wrap-resource "public" {:allow-symlinks? true}) - (oauth/wrap-oauth config) + #_ (oauth/wrap-oauth config) (middleware/wrap-defaults site-defaults) wrap-exception-handling wrap-restful-format diff --git a/src/clj/org/parkerici/enflame/sparql.clj b/src/clj/org/parkerici/enflame/sparql.clj deleted file mode 100644 index 307338e..0000000 --- a/src/clj/org/parkerici/enflame/sparql.clj +++ /dev/null @@ -1,295 +0,0 @@ -(ns org.parkerici.enflame.sparql - (:require - [clj-http.client :as client] - [arachne.aristotle.query :as q] - [arachne.aristotle.registry :as reg] - [arachne.aristotle.graph :as graph] - [clojure.data.json :as json] - [org.parkerici.multitool.core :as u] - ) - (:import (org.apache.jena.rdf.model Model Resource Literal Property Statement RDFNode)) - ) - -;;; Not clear what Aristotle does that isn't better handled by Jena SSE https://jena.apache.org/documentation/notes/sse.html - - -(defn ->sparql [bgp & {:keys [limit]}] - (let [query (-> bgp - q/build - org.apache.jena.sparql.algebra.OpAsQuery/asQuery)] - ;; TODO nany other options, try (bean query) - (when limit - (.setLimit query limit)) - (str query))) - -(comment - (reg/prefix 'ds "https://data.lacity.org/resource/zzzz-zzzz/") - (->sparql '(:bgp {:rdf/about ?e, :ds/zip_code "90001", :ds/total_population ?pop}))) - -;;; This could probably be improved -;;; see https://github.com/ajoberstar/ike.cljj -(defn make-consumer [collector] - (reify java.util.function.Consumer - (accept [& thing] - (swap! collector conj thing)))) - -;;; Note: not a Graph -(defn sparql-source - [url] - (-> (org.apache.jena.rdfconnection.RDFConnectionRemote/create) - (.destination url))) - -;;; TODO - want to try q/run, but it requires a Graph object and I can't figure it out... - -(defn do-query-raw - "Source is an org.apache.jena.rdfconnection.RDFConnectionRemoteBuilder - q is a sparql string" - [source q] - (let [collector (atom [])] - (.querySelect (.build source) q - (make-consumer collector)) - collector)) - -(defn result-binding->clj [rb] - (let [bindings (.getBinding rb) - vars (iterator-seq (.vars bindings))] - (zipmap (map #(keyword (.getVarName %)) vars) - (map #(graph/data (.get bindings %)) vars)))) - -(defn do-query [source q] - (let [results @(do-query-raw source q)] - (map result-binding->clj results))) - - -(defn xquery [& args] - (apply q/run args)) - - -;;; Reshaping tools - - -(defn entify-1 - [query-results] - (u/map-values (fn [c] (map :o c)) - (group-by :p query-results))) - - -(defn entify - "Turn a query results with :s :p :o fields into a set of maps" - [query-results] - (->> query-results - (group-by :s) - (u/map-values entify-1))) - -;;; Searching for usable endpoints - -(def sparql-dbs (clojure.data.json/read-str (slurp "/Users/mtravers/Downloads/query (1).json") - :key-fn keyword)) - - -(defn check-link - [url] - (client/head url - {:cookie-policy :standard - :trace-redirects true - :redirect-strategy :graceful})) - -(defn test-endpoint - [e] - (prn :testing e) - (check-link (:u e))) - -;;; From goddinpotty -(defn check-external-links - "Find bad external links in the background." - [] - (let [bads (atom []) - goods (atom [])] - (doseq [{:keys [u item] :as db} sparql-dbs] - (future-call - #(try - (prn :checking db) - (swap! goods conj [db (check-link u)]) - (catch Throwable e (swap! bads conj [db e]))))) - [bads goods])) - - -(comment -;;; These are some that at least respond. -("http://vulcan.cs.uga.edu/sparql" ;;; Prokino, see http://prokino.uga.edu/prokino/query/sparql - - - "http://opencitations.net/sparql" - "https://sparql.proconsortium.org/virtuoso/sparql" - "http://opendatacommunities.org/sparql" - "http://data.allie.dbcls.jp/sparql" - "https://bgee.org/sparql" - "https://ld.cultural.jp/sparql" - "http://genome.microbedb.jp/sparql" - "https://data.europa.eu/euodp/sparqlep" - "http://id.nlm.nih.gov/mesh/sparql" - "https://colil.dbcls.jp/sparql" - "http://data.archiveshub.ac.uk/sparql" - "http://lod.nl.go.kr/sparql" - "https://query.inventaire.io" - "https://sparql.uniprot.org/sparql" - "http://www.europeandataportal.eu/sparql" - "https://data.norge.no/sparql" - "http://id.sgcb.mcu.es/sparql" - "http://data.bibliotheken.nl/sparql" - "http://rdf.disgenet.org/sparql/" - "http://statistics.data.gov.uk/sparql" - "http://statistics.data.gov.uk/sparql" - "http://linkedgeodata.org/sparql" - "https://id.ndl.go.jp/auth/ndla/sparql" - "http://cultura.linkeddata.es/sparql" - "https://jpsearch.go.jp/rdf/sparql/" - "https://isidore.science/sqe" - "https://commons-query.wikimedia.org/" - "https://mediag.bunka.go.jp/madb_lab/lod/sparql/" - "http://patho.phenomebrowser.net/sparql/" - "http://sparql.archives-ouvertes.fr/sparql" - "https://www.dictionnairedesfrancophones.org/sparql" - "https://database.factgrid.de/query/" - "https://datos.gob.es/es/sparql" - "https://sparql.orthodb.org/sparql" - "https://data.gov.cz/sparql" - "http://data.archaeologydataservice.ac.uk/sparql/repositories/archives" - "https://lingualibre.org/bigdata/namespace/wdq/sparql" - "https://slod.fiz-karlsruhe.de/sparql" - "https://druid.datalegend.net/AdamNet/Heritage/sparql/Heritage" - "http://www.genome.jp/sparql/linkdb" - "http://ma-graph.org/sparql" - "http://sparql.wikipathways.org/" - "http://bnb.data.bl.uk/sparql" - "http://data.culture.fr/thesaurus/sparql" - "https://data.muziekweb.nl/MuziekwebOrganization/Muziekweb/sparql/Muziekweb" - "http://data.bnf.fr/sparql/" - "https://labs.onb.ac.at/en/tool/sparql/" - "http://dati.camera.it/sparql" - "https://tora.entryscape.net/snorql" - "https://sparql.rhea-db.org/sparql" - "http://data.nobelprize.org/" - "https://triplestore.iccu.sbn.it/sparql" - "https://triplestore.iccu.sbn.it/sparql" - "https://query.linkedopendata.eu/sparql" - "http://bio2rdf.org/sparql" - "http://vocabulary.curriculum.edu.au/PoolParty/sparql/scot" - "http://data.cervantesvirtual.com/openrdf-sesame/repositories/data" - "https://libris.kb.se/sparql" - "http://data.nationallibrary.fi/bib/sparql" - "http://data.persee.fr/sparql" - "https://data.idref.fr/sparql" - "https://data.cssz.cz/sparql" - "https://www.orpha.net/sparql" - "https://www.orpha.net/sparql" - "https://datos-abertos.galiciana.gal/pt/sparql" - "https://xn--slovnk-7va.gov.cz/sparql" - "https://idsm.elixir-czech.cz/sparql/" - "http://dbpedia.org/sparql" - "http://lod.openaire.eu/endpoint" - "http://sparql.europeana.eu/" - "http://data.ordnancesurvey.co.uk/datasets/os-linked-data/apis/sparql" - "https://query.wikidata.org/sparql") -) - - -;;; Abstracted from Uniprot -(defn ^:api q - [endpoint sparql] - (if (string? sparql) - (do-query (sparql-source endpoint) sparql) - (q endpoint (->sparql sparql)))) - - -(defn ontology-query - [endpoint] - (-> - (entify - (q endpoint - '(:bgp [?s :rdfs/isDefinedBy ?uniprot] - [?s ?p ?o]))))) - -(defn ontology-query-2 - [endpoint] - (-> - (entify - (q endpoint - '(:bgp [?s :rdf/type :owl/Class] - [?s ?p ?o]))))) - -(defn ^:api instances - "Does a pull of all instances of class" - [endpoint class] - (entify - (q endpoint - `(:bgp [?s :rdf/type ~class] - [?s ?p ?o])))) - -(defn ^:api instances-only - "Gets all instances of class" - [endpoint class] - (q endpoint - `(:bgp [?s :rdf/type ~class] - ))) - -(defn ^:api pull - [endpoint ent] - (entify-1 - (q endpoint - `(:bgp [~ent ?p ?o])) - )) - -(defn ^:api pull2 - [endpoint ent] - (entify-1 - (q endpoint - `(:conditional - (:bgp [?o ?p2 ?o2]) - (:bgp [~ent ?p ?o]))))) - - -(defn ^:api pull2 - [endpoint ent] - (q endpoint - `(:bgp [~ent ?p ?o] - [?o ?p2 ?o2]))) - -(defn ^:api pull3 - [endpoint ent] - (q endpoint - `(:bgp [~ent ?p ?o] - [?o ?p2 ?o2] - [?o2 ?p3 ?o3] - ))) - - - - -(defn ^:api antipull - [endpoint ent] - (entify-1 - (q endpoint - `(:bgp [?o ?p ~ent])) - )) - -(defn attributes - [endpoint att] - (q endpoint - `(:bgp [?s ~att ?o])) - ) - - -(comment -;;; Look here for examples -;;; https://github.com/arachne-framework/aristotle/blob/master/test/arachne/aristotle/query_test.clj#L90 -;;; This runs but seems to ignore the aggregate spec -(q '(:group (?s) ((?s (count ?s))) (:bgp [?s :rdfs/domain :prokino/LigandActivity]))) - - - -;;; Example of DISTINCT -(def x (sq/q endpoint - '(:distinct (:project [?o] (:bgp [?s :rdf/type :prokino/LigandActivity] - [?s :prokino/hasMOA ?o]))))) -) diff --git a/src/clj/org/parkerici/enflame/uniprot.clj b/src/clj/org/parkerici/enflame/uniprot.clj deleted file mode 100644 index ad4a58d..0000000 --- a/src/clj/org/parkerici/enflame/uniprot.clj +++ /dev/null @@ -1,314 +0,0 @@ -(ns org.parkerici.enflame.uniprot - (:require - [org.parkerici.enflame.sparql :as sq] - [arachne.aristotle.registry :as reg] - [org.parkerici.multitool.core :as u] - [org.parkerici.multitool.cljcore :as ju] - [clojure.set :as set] - ) ) - -;;; These are silly -(reg/prefix 'uniprot "http://purl.uniprot.org/core/") -(reg/prefix 'unipath "http://purl.uniprot.org/unipathway/") -(reg/prefix 'unicite "http://purl.uniprot.org/citations/") -(reg/prefix 'unidb "http://purl.uniprot.org/database") -(reg/prefix 'dcterms "http://purl.org/dc/terms/") -(reg/prefix 'unienzyme "http://purl.uniprot.org/enzyme/") -(reg/prefix 'skos "http://www.w3.org/2004/02/skos/core#") - -;;; Try (this does not seem to work, sigh) -(reg/prefix 'uniuni "http://purl.uniprot.org/") - -(defn uniprot-q - [sparql] - (if (string? sparql) - (sq/do-query (sq/sparql-source "https://sparql.uniprot.org/") sparql) - (uniprot-q (sq/->sparql sparql :limit 100000)))) ;TODO limit temp :limit 1000 - - -(def external-ontology - '{:rdf/Statement {:rdf/type (:owl/Class)}}) - -;;; TODO shouldn't run on compile -(defonce uniprot-ontology - (-> - (sq/entify - (uniprot-q - '(:bgp [?s :rdfs/isDefinedBy ?uniprot] - [?s ?p ?o]))) - (merge external-ontology) - ;; This one field comes back with an unserializable object, just patch it - ;; Real thing (.-lexicalValue _) if need be - (assoc-in [:uniprot/Pathway :rdfs/label] '("Pathway")))) - -;;; TODO damn I wish these were more composable - - -;;; Note :unipath/399.28.3.3 doesn't work with Clojure reader, argh -#_ -(pull (keyword "unipath" "399.28.3.3")) - - -(defn filtered-by - [field value] - (u/dissoc-if (fn [[n d]] - (not (some #(= value %) (field d)))) - uniprot-ontology)) - -(defn filtered-by-any - [field values] - (u/dissoc-if (fn [[n d]] - (empty? (set/intersection (set (field d)) (set values)))) - uniprot-ontology)) - - -(defn filtered-by-rdf-type - [type] - (u/dissoc-if (fn [[n d]] - ;; TODO assuming a single type - (not (= (:rdf/type d) (list type)))) - uniprot-ontology) ) - -(defn classes - [] - (filtered-by-rdf-type :owl/Class)) - -(defn subclasses - [class] - (filtered-by :rdfs/subClassOf class)) - -(def all-subclasses - (u/transitive-closure (comp keys subclasses))) - -(defn properties - [] - (merge (filtered-by-rdf-type :owl/DatatypeProperty) - (filtered-by-rdf-type :owl/ObjectProperty))) - -(defn properties-for-domain - [class] - (filtered-by-any :rdfs/domain (all-subclasses class))) - - - - - - - -(defn uniprot? - [ent] - (and (keyword? ent) - (= "uniprot" (namespace ent)))) - -(defn top-classes - [] - (filter (fn [[c d]] - (not (contains? (classes) (:rdfs/subClassOf d)))) - (classes))) - -(defn top-classes - [] - (let [non-tops (keys (filtered-by-any :rdfs/subClassOf (keys (classes))))] - (apply dissoc (classes) non-tops))) - -#_ -(:uniprot/Database - :uniprot/Structured_Name - :uniprot/Enzyme_Regulation_Annotation - :uniprot/Enzyme - :uniprot/Excluded_Proteome - :uniprot/Gene - :uniprot/Citation - :uniprot/Attribution - :uniprot/Organelle - :uniprot/Status - :uniprot/Part - :uniprot/Participant - :uniprot/Journal - :uniprot/Structure_Mapping_Statement - :uniprot/Proteome - :uniprot/Nucleotide_Mapping_Statement - :uniprot/Method - :uniprot/Taxon - :uniprot/Molecule - :uniprot/Obsolete - :uniprot/Disease - :uniprot/Resource - :uniprot/Proteome_Component - :uniprot/Cluster - :uniprot/Domain_Assignment_Statement - :uniprot/Protein_Existence - :uniprot/Subcellular_Location - :uniprot/Transposon - :uniprot/Plasmid - :uniprot/Concept - :uniprot/Annotation - :uniprot/Endpoint_Statement - :uniprot/Protein - :uniprot/Tissue - :uniprot/Sequence - :uniprot/Strain - :uniprot/Interaction - :uniprot/Catalytic_Activity - :uniprot/Not_Obsolete - :uniprot/Pathway - :uniprot/Citation_Statement - :uniprot/Rank - :uniprot/Reviewed) - -(count (instances :uniprot/Pathway)) -3117 -(count (instances :uniprot/Disease)) -6202 -(count (instances :uniprot/Molecule)) -0 ;; uh uo - - -(defn describe - [ent] - (concat - (uniprot-q `(:bgp [~ent ?p ?o])) - (uniprot-q `(:bgp [?s ?p ~ent])))) - - -(comment -(frequencies (map :rdf/type (vals uniprot-ontology) )) -{(:owl/DatatypeProperty) 43, - (:owl/ObjectProperty) 67, - (:owl/NamedIndividual :owl/Thing :uniprot/Organelle) 9, - (:owl/NamedIndividual :owl/Thing :uniprot/Rank) 31, - (:owl/Class) 168, - (:owl/InverseFunctionalProperty :owl/FunctionalProperty :owl/ObjectProperty) 1, - (:owl/FunctionalProperty :owl/ObjectProperty) 6, - (:owl/FunctionalProperty :owl/DatatypeProperty) 31, - (:owl/NamedIndividual :owl/Thing :uniprot/Status) 4, - (:owl/NamedIndividual :owl/Thing :uniprot/Protein_Existence) 5, - (:owl/NamedIndividual :owl/Thing :uniprot/Mass_Measurement_Method) 7, - (:owl/NamedIndividual :owl/Thing :uniprot/Structure_Determination_Method) 7} -) - - - -#_ -(frequencies (map :rdfs/domain (vals (properties)))) - -;;; Huh. -#_ -{nil 17, - (:uniprot/Subcellular_Location_Annotation) 1, - (:uniprot/Structured_Name) 1, - (:uniprot/Proteome) 2, - (:uniprot/Interaction) 1, - (:uniprot/Reviewed_Protein) 1, - (_626915beb033654fc13c8409d68fbefb) 1, - (:uniprot/RNA_Editing_Annotation) 1, - (:uniprot/External_Sequence) 1, - (_135e6ace0ba508ab2319c50063cc0ede) 1, - (_7a99d05307375d434d4a3f01c938cad7) 1, - (_b868ac3eb7960429cb539cea5f6300ae) 1, - (:uniprot/Gene) 2, - (_ade505809c0086211dc02c0f9464258e) 1, - (:uniprot/Resource) 1, - (_cfd54a0e1d32a8d62b76f3490d7f2311) 1, - (:uniprot/Transcript_Resource) 2, - (:uniprot/Published_Citation) 2, - (_e978c46c81fd658d3e99796c5eaf4502) 1, - (_5fcb391e3136fc76ba988a8b5f961505) 1, - (_941b9dd17dd300caedb9cb8c0e3f958e) 1, - (:uniprot/Disease_Annotation) 1, - (_d0e6352838bda96a5d57075fed61b8fc) 1, - (:uniprot/Simple_Sequence) 2, - (:uniprot/Protein) 13, - (:uniprot/Modified_Sequence) 1, - (:uniprot/Enzyme) 2, - (:uniprot/Catalytic_Activity_Annotation) 2, - (_49396fea4d19b3d5b39285cc1252056b) 1, - (_e2daa4cde5ccb48ceaa946a7c97ec83e) 1, - (:uniprot/Journal) 1, - (_3725fc1a96109bd014937233e0cc1e80) 1, - (:uniprot/Cluster) 2, - (_086cb0592bb819a0cd93d44d3bf577d8) 1, - (:uniprot/Binding_Site_Annotation) 2, - (_63f5f671dd82a005a9e91a5c22c458a5) 1, - (:uniprot/Structure_Mapping_Statement) 1, - (_a2745b16016d308213a790963118d9a8) 1, - (:rdfs/Resource) 1, - (:uniprot/Thesis_Citation) 1, - (_d84a5a698f10400d9ffba7376653fc21) 1, - (:uniprot/Nucleotide_Resource) 2, - (_9d3897b13bb29c76ec292b254542decb) 1, - (:uniprot/Subcellular_Location) 1, - (_612e2c86654093537c55009db223f480) 1, - (:uniprot/Book_Citation) 2, - (:uniprot/Kinetics_Annotation) 2, - (:uniprot/Sequence) 3, - (:uniprot/Cofactor_Annotation) 1, - (:uniprot/Catalytic_Activity) 1, - (:uniprot/Citation) 5, - (:uniprot/Database) 5, - (:uniprot/Submission_Citation) 1, - (:uniprot/Taxon) 4, - (:uniprot/Attribution) 1, - (:uniprot/Citation_Statement) 2} - - -;;; Alzabo schema gen - -;;; TODO should include the real URI somewhere - -;;; TODO Alzabo has no concept of subclass, would be interesting to add -;;; For now, it compresses everything into top classes - -;;; Remove namespace (see u/d-ns) -(defn nons - [key] - (if (keyword key) - (keyword (name key)) - key)) - -;;; TODO add skos etc fields -(defn class-alzabo-fields - [class] - (apply - merge - (for [[n d] (properties-for-domain class)] - {(nons n) - {:type (or (nons (first (:rdfs/range d))) - :string) ;temp but nil doen't work - ;; :cardinality ... - :uri n - :attribute n ;aka :uri, but this leverages existing mechanisms - :doc (first (:rdfs/comment d))}} - ))) - -(defn alzabo - [] - (u/clean-walk - {:title "UNIPROT" - :kinds - (apply - merge - (for [[tc tc-def] (top-classes)] - {(nons tc) - {:doc (first (:rdfs/comment tc-def)) - :title (first (:rdfs/label tc-def)) ;not actually used or defined - :fields (or (class-alzabo-fields tc) {}) - :uri tc - }}))} - nil?)) - -#_ -(ju/schppit "uniprot-ontology.edn" uniprot-ontology) - -#_ -(ju/schppit "resources/uniprot-alzabo.edn" (alzabo)) - - -;;; Regex usage - - - - `(:bgp [?protein :rdf/type :uniprot/Protein] - [?protein :uniprot/classifiedWith ?concept] - [?concept :rdfs/label ?clabel] - [(regex ?clabel "FOO.*" "")]) diff --git a/src/cljc/org/parkerici/enflame/blockdefs.cljc b/src/cljc/org/parkerici/enflame/blockdefs.cljc index f06203a..0f3c75a 100644 --- a/src/cljc/org/parkerici/enflame/blockdefs.cljc +++ b/src/cljc/org/parkerici/enflame/blockdefs.cljc @@ -150,7 +150,7 @@ {:message0 (str (name field) "? %1") :args0 [{:name "V" :type "field_dropdown" - :options [["yes" :true] ["no" :false] ["defined" :any]]}] + :options '[["yes" "true"] ["no" "false"] ["defined" "any"]]}] :query-builder :query-primitive-field} (:long :float) {:message0 (str (name field) " %2 %1") @@ -220,23 +220,42 @@ (defn field-def [kind field] (get-in (schema/kind-def kind) [:fields field])) -(defn kind-field-blockdef - [kind field invert?] - (let [{:keys [type attribute]} (if invert? ;??? not sure about this - {:type field :attribute (keyword field kind)} - (field-def kind field))] + +(defn base-block + [kind field attribute] + {:type (str (name kind) "_" (name field)) + :previousStatement (str (name kind) "_constraint") + :nextStatement (str (name kind) "_constraint") + :colour (kind-color kind) + :helpUrl (alzabo-url kind) + :attribute (or attribute (keyword (name kind) (name field))) + :input (or (:type (field-def kind field)) + field) + }) + + +;;; eg :measurement :measurement-set {:type :measurement-set, :cardinality :many, :doc "The measurement values for this measurement set", :attribute :measurement-set/measurements} +(defn kind-field-inverse-blockdef + [kind {:keys [type attribute]}] + (let [field type] + (merge + {:invert? true} + (field-def-type field field) + (base-block kind field attribute)))) + + +(defn kind-field-forward-blockdef + [kind field] + ;; TODO this is wrong, some attributes are plural eg :measurement-set/measurements + (let [{:keys [type attribute]} (field-def kind field)] (when-let [field-def (field-def-type field type)] (merge field-def - {:type (str (name kind) "_" (name field)) - :previousStatement (str (name kind) "_constraint") - :nextStatement (str (name kind) "_constraint") - :colour (kind-color kind) - :helpUrl (alzabo-url kind) - :attribute (or attribute (keyword (name kind) (name field))) - :input type - :invert? invert? - })))) + (base-block kind field attribute) + )))) + + + ;;; ⊓⊔⊓⊔ Complex relations ⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔ @@ -354,12 +373,11 @@ (concat ;; forward relations (map (fn [field] - (kind-field-blockdef kind field false)) + (kind-field-forward-blockdef kind field)) (keys (schema/kind-fields kind))) ;; inverse relations - (map (fn [[other-kind adef]] - (let [adef (first adef)] - (kind-field-blockdef kind other-kind true))) + (map (fn [[other-kind [adef]]] + (kind-field-inverse-blockdef kind adef)) (kind (schema/inverse-relations))) (kind-complex-blockdefs kind)) (filter identity) diff --git a/src/cljc/org/parkerici/enflame/library/item.cljc b/src/cljc/org/parkerici/enflame/library/item.cljc index f63af8d..aee86e4 100644 --- a/src/cljc/org/parkerici/enflame/library/item.cljc +++ b/src/cljc/org/parkerici/enflame/library/item.cljc @@ -24,7 +24,7 @@ [item] (or (::label item) (::description item) (::query item))) -(def big-keys #{::blockdef ::image}) +#_(def big-keys #{::blockdef ::image}) ;;; Convert from gcs diff --git a/src/cljc/org/parkerici/enflame/schema.cljc b/src/cljc/org/parkerici/enflame/schema.cljc index fd8e1b5..6bef1c7 100644 --- a/src/cljc/org/parkerici/enflame/schema.cljc +++ b/src/cljc/org/parkerici/enflame/schema.cljc @@ -36,6 +36,10 @@ [k] (get-in @the-schema [:kinds k :fields])) +(defn kind-field + [k f] + (get-in @the-schema [:kinds k :fields f])) + (defn kind? [k] (contains? (:kinds @the-schema) k)) diff --git a/src/cljs/org/parkerici/enflame/config.cljs b/src/cljs/org/parkerici/enflame/config.cljs index 0439db5..5c535ce 100644 --- a/src/cljs/org/parkerici/enflame/config.cljs +++ b/src/cljs/org/parkerici/enflame/config.cljs @@ -22,7 +22,7 @@ (defonce the-config (atom nil)) (defn config - ([key] (get @the-config key)) + ([& keys] (get-in @the-config keys)) ([] @the-config )) ;;; Get the config from the server diff --git a/src/cljs/org/parkerici/enflame/core.cljs b/src/cljs/org/parkerici/enflame/core.cljs index 9aa9019..c0b9b41 100644 --- a/src/cljs/org/parkerici/enflame/core.cljs +++ b/src/cljs/org/parkerici/enflame/core.cljs @@ -1,4 +1,3 @@ - (ns org.parkerici.enflame.core (:require [reagent.dom :as rdom] @@ -9,7 +8,7 @@ [org.parkerici.enflame.views :as views] [org.parkerici.enflame.db] [org.parkerici.enflame.config :as config] - org.parkerici.enflame.embed + [org.parkerici.enflame.embed :as embed] org.parkerici.enflame.schema-client )) @@ -68,6 +67,23 @@ (rf/dispatch [:dispatch-when :schema [:library-load library]]))) ) +;;; This is identitical to :candel (for now) +(defmethod custom-init :datomic-cloud + [_] + (rf/dispatch [:get-ddbs]) + (let [{:keys [library ddb query] :as _params} (browser/url-params)] + (if-let [ddb (or ddb (config/get-local :ddb))] + (rf/dispatch [:set-ddb ddb]) + (rf/dispatch [:set-schema])) + (when query + ;; Wait for schema to be set + (rf/dispatch [:dispatch-when :schema [:set-query query]])) + ;; NOt really CANDEL specifica + (when library + ;; Wait for schema to be set + (rf/dispatch [:dispatch-when :schema [:library-load library]]))) + ) + (defn ^:export init [] @@ -79,3 +95,12 @@ (ag/init) ) +;;; This used to be in embed but that has dependency issues +(defn ^:export embed + [] + (config/init #(do + (custom-init (config/config)) + (embed/re-frame-init) + (embed/init-embed) + )) + (ag/init)) diff --git a/src/cljs/org/parkerici/enflame/datomic.cljs b/src/cljs/org/parkerici/enflame/datomic.cljs index dd45a42..621edfe 100644 --- a/src/cljs/org/parkerici/enflame/datomic.cljs +++ b/src/cljs/org/parkerici/enflame/datomic.cljs @@ -3,9 +3,20 @@ [org.parkerici.multitool.core :as u] [org.parkerici.enflame.api :as api] [org.parkerici.enflame.results :as results] + [clojure.walk :as walk] [cljs.reader :as reader] )) +;;; Note: for some reason, transit is producing goog.long objectts instead of proper clojure longs. Can't figure out why, this patches the problem +(defn fix-results + [r] + (walk/postwalk + (fn [x] + (if (.-low_ x) + (long x) + x)) + r)) + ;;; TODO this isn't all that Datomic specific, maybe rename @@ -21,7 +32,8 @@ (api/ajax-get "/api/query" (merge {:url-params (u/clean-map {:db ddb :query (str query) :limit limit :args (str args)}) - :handler handler} + :handler (fn [r] + (handler (fix-results r)))} (or options {})))) @@ -40,8 +52,9 @@ nil ;no limit (fn [{:keys [results _count _clipped]}] ;; TODO maybe filter out the Datomic bookkeeping ones - (rf/dispatch [:set-idents ddb (zipmap (map first results) - (map (comp reader/read-string second) results))])) + #_(zipmap (map first results) + (map (comp reader/read-string second) results)) + (rf/dispatch [:set-idents ddb (into {} results)])) )) db)) diff --git a/src/cljs/org/parkerici/enflame/embed.cljs b/src/cljs/org/parkerici/enflame/embed.cljs index f76b004..81b5a37 100644 --- a/src/cljs/org/parkerici/enflame/embed.cljs +++ b/src/cljs/org/parkerici/enflame/embed.cljs @@ -8,6 +8,7 @@ [org.parkerici.enflame.views :as views] [org.parkerici.enflame.view.graph :as graph] [org.parkerici.enflame.db] + [org.parkerici.enflame.config :as config] )) ;;; Alternative to core for embedded @@ -47,15 +48,14 @@ "data" views/results}) (views/results))]) -(defn ^:export re-frame-init +(defn re-frame-init [] (rf/dispatch-sync [:initialize-db]) (rf/clear-subscription-cache!)) ;; Probably this would be better if blockly was a real react component. -(defn ^:export init +(defn init-embed [] - (re-frame-init) (let [{:keys [ddb query view rows] :as _params} (browser/url-params)] (let [blockly-id "blocklyEmbed" results-id "results" diff --git a/src/cljs/org/parkerici/enflame/view/candel_cards.cljs b/src/cljs/org/parkerici/enflame/view/candel_cards.cljs index 2f7c492..a41df0f 100644 --- a/src/cljs/org/parkerici/enflame/view/candel_cards.cljs +++ b/src/cljs/org/parkerici/enflame/view/candel_cards.cljs @@ -15,14 +15,15 @@ ;;; Hm. Haven't really been designing with that in mind but it could work. (defn db-card [] [vu/card "DB" - [:div - [:select.form-control - {:name "ddb" - :style {:width "100%"} - :on-change #(rf/dispatch [:set-ddb (-> % .-target .-value)]) - :value (or @(rf/subscribe [:ddb]) "")} - (for [db @(rf/subscribe [:ddbs])] - [:option {:key db :value db} db])]]]) + [:div + [:select.form-control + {:name "ddb" + :style {:width "100%"} + :on-change #(rf/dispatch [:set-ddb (-> % .-target .-value)]) + :value (or @(rf/subscribe [:ddb]) "")} + (for [db @(rf/subscribe [:ddbs])] + [:option {:key db :value db} db])]] + :header-extra [:span.float-right2.h4 @(rf/subscribe [:ddb])]]) (defn wick-card [] diff --git a/src/cljs/org/parkerici/enflame/views.cljs b/src/cljs/org/parkerici/enflame/views.cljs index 2980e32..3c16de3 100644 --- a/src/cljs/org/parkerici/enflame/views.cljs +++ b/src/cljs/org/parkerici/enflame/views.cljs @@ -173,8 +173,11 @@ (defn error [[status response]] - [:div {:style {:color "red"}} - (str status ": " response)]) + [:div + [:h3 "Error"] + [:pre {:style {:color "red"}} + (with-out-str (pprint/pprint status))] + ]) (defn download-link [] @@ -330,11 +333,14 @@ [:div {:style {:margin-top "10px"}} [:div ;; TODO customize these - (toplink [:span [:b "Enflame "] [:img {:src "favicon.ico" :width "24px"}]] "http://github.com/ParkerICI/enflame") + (toplink [:span [:b "Enflame "] [:img {:src "favicon.ico" :width "24px"}]] "http://github.com/CandelBio/enflame") (toplink "Tutorial" "/doc/tutorial.html") (toplink "Doc" "/doc/guide.html") - (toplink "Schema" (str "/schema/" "" "/index.html")) ;TODO version - (toplink "Library" "/library")] + (toplink "Schema" (str "/schema/index.html")) ;TODO version + + (when (c/config :library :type) + (toplink "Library" "/library")) + ] [:div#accordian.accordian (for [card (c/config :rh-cards)] ;; cards TODO not working yet because timing diff --git a/test/clj/org/parkerici/enflame/blockdefs_test.clj b/test/clj/org/parkerici/enflame/blockdefs_test.clj index f192f8e..c53ae33 100644 --- a/test/clj/org/parkerici/enflame/blockdefs_test.clj +++ b/test/clj/org/parkerici/enflame/blockdefs_test.clj @@ -5,7 +5,7 @@ [org.parkerici.enflame.schema :as schema] )) -(use-fixtures :once with-schema) +(use-fixtures :once with-test-config) (deftest blockdef-test (testing "blockdefs for single kind (subject)" diff --git a/test/clj/org/parkerici/enflame/library_test.clj b/test/clj/org/parkerici/enflame/library_test.clj index fef8f7a..e9dbf25 100644 --- a/test/clj/org/parkerici/enflame/library_test.clj +++ b/test/clj/org/parkerici/enflame/library_test.clj @@ -7,7 +7,7 @@ [clojure.test :as t])) -(use-fixtures :once with-schema) +(use-fixtures :once with-test-config) ;;; Turn off annoying print-map feature (defmethod print-method clojure.lang.IPersistentMap [m, ^java.io.Writer w] @@ -53,6 +53,7 @@ :dquery {:find ((pull ?subject1 [:db/id :subject/id])), :where ([?subject1 :subject/id ?id1])} } + {:query "[samples where [tumor-type is *]]", :compact {:type "sample_query", @@ -88,8 +89,6 @@ :where ([?subject1 :subject/meddra-disease ?meddra-disease1])}} - - {:query "[samples where [gdc-anatomic-site is [gdc-anatomic-sites]]]", :compact {:type "sample_query", diff --git a/test/clj/org/parkerici/enflame/query_test.clj b/test/clj/org/parkerici/enflame/query_test.clj index 8197438..1f9bf07 100644 --- a/test/clj/org/parkerici/enflame/query_test.clj +++ b/test/clj/org/parkerici/enflame/query_test.clj @@ -3,7 +3,7 @@ [org.parkerici.enflame.test-utils :refer :all] [org.parkerici.enflame.candel.query :refer :all])) -(use-fixtures :once with-schema) +(use-fixtures :once with-test-config) (defn set= [a b] (and (= (set a) (set b)) diff --git a/test/clj/org/parkerici/enflame/test_utils.clj b/test/clj/org/parkerici/enflame/test_utils.clj index c942d19..730180c 100644 --- a/test/clj/org/parkerici/enflame/test_utils.clj +++ b/test/clj/org/parkerici/enflame/test_utils.clj @@ -1,14 +1,15 @@ (ns org.parkerici.enflame.test-utils (:require [clojure.test :refer :all] - [org.parkerici.enflame.server :as server] - [org.parkerici.enflame.schema :as schema] [org.parkerici.enflame.config :as config] )) -(use-fixtures :once - (config/load-config "test/resources/test-config.edn")) +(defn with-test-config + [f] + (config/load-config "test/test-config.edn") + (f)) + +;;; Put this in individual test files +#_ (use-fixtures :once with-test-config) + -(defn with-schema [f] - (schema/set-schema (config/read-schema nil nil)) ;TODO args - (f))