From 2f77651139ce1e206593477ec97b632aa1276316 Mon Sep 17 00:00:00 2001 From: Mike Travers Date: Fri, 10 Feb 2023 16:37:25 -0800 Subject: [PATCH 01/19] Upgraded ring to avoid dependency conflicts with Datomic --- project.clj | 30 ++++++++++++++++-------------- resources/opencandel-config.edn | 25 +++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 14 deletions(-) create mode 100644 resources/opencandel-config.edn diff --git a/project.clj b/project.clj index ba0fc8f..c1901e7 100644 --- a/project.clj +++ b/project.clj @@ -1,25 +1,27 @@ (defproject enflame "0.0.29-SNAPSHOT" - :dependencies [[org.clojure/clojure "1.10.1"] - [clj-http "3.12.1" + :dependencies [[org.clojure/clojure "1.11.1"] + [clj-http "3.12.3" :exclusions [commons-codec]] - [cheshire "5.10.0"] + [cheshire "5.11.0"] + + [com.datomic/client-cloud "1.0.122"] ;; Ring and its large family - [org.eclipse.jetty/jetty-client "9.4.12.v20180830"] ;has to match ring version of jetty - [org.eclipse.jetty/jetty-server "9.4.12.v20180830"] - [org.eclipse.jetty/jetty-http "9.4.12.v20180830"] - [org.eclipse.jetty/jetty-util "9.4.12.v20180830"] + [org.eclipse.jetty/jetty-client "9.4.48.v20220622"] ;has to match ring version of jetty + [org.eclipse.jetty/jetty-server "9.4.48.v20220622"] + [org.eclipse.jetty/jetty-http "9.4.48.v20220622"] + [org.eclipse.jetty/jetty-util "9.4.48.v20220622"] - [ring "1.8.0"] - [ring/ring-jetty-adapter "1.7.1"] - [ring/ring-defaults "0.3.2"] - [ring-logger "1.0.1"] - [ring-oauth2 "0.1.4"] + [ring "1.9.6"] + [ring/ring-jetty-adapter "1.9.6"] + [ring/ring-defaults "0.3.4"] + [ring-logger "1.1.1"] + [ring-oauth2 "0.2.0" :exclusions [org.apache.httpcomponents/httpcore]] [org.slf4j/slf4j-simple "1.7.26"] ;required to turn off warning [com.taoensso/timbre "4.10.0"] [org.clojure/data.csv "0.1.4"] [compojure "1.6.1" :exclusions [ring.core ring.codec]] - [ring-middleware-format "0.7.4" :exclusions [javax.xml.bind/jaxb-api]] + [ring-middleware-format "0.7.5" :exclusions [javax.xml.bind/jaxb-api]] [bk/ring-gzip "0.3.0"] [trptcolin/versioneer "0.2.0"] [com.google.cloud/google-cloud-datastore "1.105.7" @@ -31,7 +33,7 @@ com.fasterxml.jackson.core/jackson-core]] [environ "1.1.0"] [me.raynes/fs "1.4.6"] - [org.parkerici/multitool "0.0.19"] + [org.parkerici/multitool "0.0.26"] [com.cemerick/url "0.1.1"] [org.clojure/data.xml "0.2.0-alpha6"] [org.clojure/clojurescript "1.10.520"] diff --git a/resources/opencandel-config.edn b/resources/opencandel-config.edn new file mode 100644 index 0000000..59f24b6 --- /dev/null +++ b/resources/opencandel-config.edn @@ -0,0 +1,25 @@ +{:source {:type :datomic-cloud ;The data source. + :config {:server-type :ion ;Passed directly to Datomic Client API + :region "us-east-1" ;; e.g. us-east-1 + :system "PublicCANDEL" + ;; :creds-profile "" + :endpoint "https://pepaga6o2e.execute-api.us-east-1.amazonaws.com/"} + } + + :schema "resources/candel-schema-1-3-1.edn" ;filename of an Alzabo schema + + :library {:gcs-project :redacted ;GCS Datastore project for storiing library entries + :gcs-console-url :redacted} + + :query-generator :candel-generate ;The query generator + + :rh-cards [:candel/db ;Sequence of cards to show in right-hand side panel + :query + :candel/wick + :share + :compact ;debug only + :browser + ] + :port 1997 ;Port on which to serve application + :dev? true ;debug only + } From b420504c21178709986b186ccb55cd963af1fc42 Mon Sep 17 00:00:00 2001 From: Mike Travers Date: Sat, 11 Feb 2023 15:18:06 -0800 Subject: [PATCH 02/19] queries to opencandel (datomic cloud client api) working! --- resources/opencandel-config.edn | 4 +++- .../org/parkerici/enflame/datomic_client.clj | 18 ++++++++++++++++++ src/clj/org/parkerici/enflame/server.clj | 4 +++- src/cljs/org/parkerici/enflame/core.cljs | 17 +++++++++++++++++ src/cljs/org/parkerici/enflame/datomic.cljs | 5 +++-- 5 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 src/clj/org/parkerici/enflame/datomic_client.clj diff --git a/resources/opencandel-config.edn b/resources/opencandel-config.edn index 59f24b6..439647e 100644 --- a/resources/opencandel-config.edn +++ b/resources/opencandel-config.edn @@ -4,6 +4,7 @@ :system "PublicCANDEL" ;; :creds-profile "" :endpoint "https://pepaga6o2e.execute-api.us-east-1.amazonaws.com/"} + :db-name "pici0044-5" } :schema "resources/candel-schema-1-3-1.edn" ;filename of an Alzabo schema @@ -15,7 +16,8 @@ :rh-cards [:candel/db ;Sequence of cards to show in right-hand side panel :query - :candel/wick + ;; I guess no point to this until Wick server is brought up + ;; :candel/wick :share :compact ;debug only :browser diff --git a/src/clj/org/parkerici/enflame/datomic_client.clj b/src/clj/org/parkerici/enflame/datomic_client.clj new file mode 100644 index 0000000..92a4f1d --- /dev/null +++ b/src/clj/org/parkerici/enflame/datomic_client.clj @@ -0,0 +1,18 @@ +(ns org.parkerici.enflame.datomic-client + (:require [datomic.client.api :as d] + [taoensso.timbre :as log] + [org.parkerici.multitool.core :as u] + [org.parkerici.enflame.config :as config] + )) + +;;; TODO do these at a reasonable time + +(def client (d/client (config/config :source :config))) + +;;; OK probably we want to handle multiple databases +(def conn (d/connect client {:db-name "pici0044-5"})) ;; (config/config :source :db-name)})) + +(defn query + [db query args] + (prn :query db query args) + (d/q {:query query :args (cons (d/db conn) args)})) diff --git a/src/clj/org/parkerici/enflame/server.clj b/src/clj/org/parkerici/enflame/server.clj index 8c1463b..4563ced 100644 --- a/src/clj/org/parkerici/enflame/server.clj +++ b/src/clj/org/parkerici/enflame/server.clj @@ -1,5 +1,6 @@ (ns org.parkerici.enflame.server (:require [org.parkerici.enflame.datomic-relay :as datomic] + [org.parkerici.enflame.datomic-client :as datomic-client] [org.parkerici.enflame.download :as download] [org.parkerici.enflame.embed-server :as embed] [org.parkerici.enflame.admin :as admin] @@ -43,7 +44,8 @@ _args (if (u/nullish? args) [] (read-string args)) _limit (if (u/nullish? limit) nil (Integer. limit)) candelabra-token (get-in req [:cookies "candelabra-token" :value]) - results (datomic/query db _query _args candelabra-token config) + results #_ (datomic/query db _query _args candelabra-token config) + (datomic-client/query db _query _args) clipped (if _limit (take _limit results) results)] (response/response {:count (count results) :clipped (count clipped) :results clipped}))) diff --git a/src/cljs/org/parkerici/enflame/core.cljs b/src/cljs/org/parkerici/enflame/core.cljs index 9aa9019..88a1b06 100644 --- a/src/cljs/org/parkerici/enflame/core.cljs +++ b/src/cljs/org/parkerici/enflame/core.cljs @@ -68,6 +68,23 @@ (rf/dispatch [:dispatch-when :schema [:library-load library]]))) ) +;;; This is identitical to :candel (for now) +(defmethod custom-init :datomic-cloud + [_] + (rf/dispatch [:get-ddbs]) + (let [{:keys [library ddb query] :as _params} (browser/url-params)] + (if-let [ddb (or ddb (config/get-local :ddb))] + (rf/dispatch [:set-ddb ddb]) + (rf/dispatch [:set-schema])) + (when query + ;; Wait for schema to be set + (rf/dispatch [:dispatch-when :schema [:set-query query]])) + ;; NOt really CANDEL specifica + (when library + ;; Wait for schema to be set + (rf/dispatch [:dispatch-when :schema [:library-load library]]))) + ) + (defn ^:export init [] diff --git a/src/cljs/org/parkerici/enflame/datomic.cljs b/src/cljs/org/parkerici/enflame/datomic.cljs index dd45a42..789114e 100644 --- a/src/cljs/org/parkerici/enflame/datomic.cljs +++ b/src/cljs/org/parkerici/enflame/datomic.cljs @@ -40,8 +40,9 @@ nil ;no limit (fn [{:keys [results _count _clipped]}] ;; TODO maybe filter out the Datomic bookkeeping ones - (rf/dispatch [:set-idents ddb (zipmap (map first results) - (map (comp reader/read-string second) results))])) + #_(zipmap (map first results) + (map (comp reader/read-string second) results)) + (rf/dispatch [:set-idents ddb (into {} results)])) )) db)) From d556d406ffcb2fcde38d7070f01061cae6661899 Mon Sep 17 00:00:00 2001 From: Mike Travers Date: Mon, 13 Feb 2023 11:29:39 -0800 Subject: [PATCH 03/19] list-databases working --- resources/opencandel-config.edn | 1 - src/clj/org/parkerici/enflame/datomic_client.clj | 14 ++++++++++---- src/clj/org/parkerici/enflame/server.clj | 7 +++++++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/resources/opencandel-config.edn b/resources/opencandel-config.edn index 439647e..52659ab 100644 --- a/resources/opencandel-config.edn +++ b/resources/opencandel-config.edn @@ -4,7 +4,6 @@ :system "PublicCANDEL" ;; :creds-profile "" :endpoint "https://pepaga6o2e.execute-api.us-east-1.amazonaws.com/"} - :db-name "pici0044-5" } :schema "resources/candel-schema-1-3-1.edn" ;filename of an Alzabo schema diff --git a/src/clj/org/parkerici/enflame/datomic_client.clj b/src/clj/org/parkerici/enflame/datomic_client.clj index 92a4f1d..c808125 100644 --- a/src/clj/org/parkerici/enflame/datomic_client.clj +++ b/src/clj/org/parkerici/enflame/datomic_client.clj @@ -7,12 +7,18 @@ ;;; TODO do these at a reasonable time -(def client (d/client (config/config :source :config))) +(u/def-lazy client (d/client (config/config :source :config))) ;;; OK probably we want to handle multiple databases -(def conn (d/connect client {:db-name "pici0044-5"})) ;; (config/config :source :db-name)})) +(u/defn-memoized conn + [db] + (d/connect @client {:db-name db})) (defn query [db query args] - (prn :query db query args) - (d/q {:query query :args (cons (d/db conn) args)})) + (d/q {:query query :args (cons (d/db (conn db)) args)})) + +(defn dbs + [] + (d/list-databases @client {})) + diff --git a/src/clj/org/parkerici/enflame/server.clj b/src/clj/org/parkerici/enflame/server.clj index 4563ced..b28d2ea 100644 --- a/src/clj/org/parkerici/enflame/server.clj +++ b/src/clj/org/parkerici/enflame/server.clj @@ -99,11 +99,18 @@ "candelabra-token" user-creds)] respon)))) +;;; Old CANDEL +#_ (defn handle-databases [req config] (let [candelabra-token (get-in req [:cookies "candelabra-token" :value])] (datomic/dbs candelabra-token config))) +;;; Open CANDSL +(defn handle-databases + [req config] + (datomic-client/dbs)) + (defn app-routes [config] (routes From 40130820c08951a13ea762974943080ac16dd1e0 Mon Sep 17 00:00:00 2001 From: Mike Travers Date: Mon, 13 Feb 2023 14:06:02 -0800 Subject: [PATCH 04/19] fixed error handling, dep problem. Also improved error UI --- project.clj | 8 ++++++-- src/clj/org/parkerici/enflame/server.clj | 4 ++-- .../parkerici/enflame/view/candel_cards.cljs | 17 +++++++++-------- src/cljs/org/parkerici/enflame/views.cljs | 7 +++++-- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/project.clj b/project.clj index c1901e7..086ec12 100644 --- a/project.clj +++ b/project.clj @@ -18,7 +18,9 @@ [ring-logger "1.1.1"] [ring-oauth2 "0.2.0" :exclusions [org.apache.httpcomponents/httpcore]] [org.slf4j/slf4j-simple "1.7.26"] ;required to turn off warning - [com.taoensso/timbre "4.10.0"] + [io.aviso/pretty "1.3"] + [com.taoensso/timbre "4.10.0" + :exclusions [io.aviso/pretty]] [org.clojure/data.csv "0.1.4"] [compojure "1.6.1" :exclusions [ring.core ring.codec]] [ring-middleware-format "0.7.5" :exclusions [javax.xml.bind/jaxb-api]] @@ -45,7 +47,9 @@ :exclusions [cljsjs/vega ; we insert a later version of Vega to fix some bugs cljsjs/vega-lite cljsjs/vega-embed - cljsjs/vega-tooltip]] + cljsjs/vega-tooltip + com.taoensso/encore]] + [cljsjs/vega "5.20.2-0"] [cljsjs/vega-lite "5.1.1-0"] [cljsjs/vega-embed "6.19.0-0"] diff --git a/src/clj/org/parkerici/enflame/server.clj b/src/clj/org/parkerici/enflame/server.clj index b28d2ea..8d1b254 100644 --- a/src/clj/org/parkerici/enflame/server.clj +++ b/src/clj/org/parkerici/enflame/server.clj @@ -150,7 +150,7 @@ (def site-defaults (-> middleware/site-defaults (assoc-in [:security :anti-forgery] false) ;interfering with save? - (assoc-in [:session :cookie-attrs :same-site] :lax) ;for oauth + #_ (assoc-in [:session :cookie-attrs :same-site] :lax) ;for oauth (assoc-in [:session :store] common-store) (assoc-in [:static :resources] nil))) ;this needs to go after oauth @@ -168,7 +168,7 @@ (when-not (some #(= % (:uri message)) no-log) (log/log level throwable message)))}) (wrap-resource "public" {:allow-symlinks? true}) - (oauth/wrap-oauth config) + #_ (oauth/wrap-oauth config) (middleware/wrap-defaults site-defaults) wrap-exception-handling wrap-restful-format diff --git a/src/cljs/org/parkerici/enflame/view/candel_cards.cljs b/src/cljs/org/parkerici/enflame/view/candel_cards.cljs index 2f7c492..a41df0f 100644 --- a/src/cljs/org/parkerici/enflame/view/candel_cards.cljs +++ b/src/cljs/org/parkerici/enflame/view/candel_cards.cljs @@ -15,14 +15,15 @@ ;;; Hm. Haven't really been designing with that in mind but it could work. (defn db-card [] [vu/card "DB" - [:div - [:select.form-control - {:name "ddb" - :style {:width "100%"} - :on-change #(rf/dispatch [:set-ddb (-> % .-target .-value)]) - :value (or @(rf/subscribe [:ddb]) "")} - (for [db @(rf/subscribe [:ddbs])] - [:option {:key db :value db} db])]]]) + [:div + [:select.form-control + {:name "ddb" + :style {:width "100%"} + :on-change #(rf/dispatch [:set-ddb (-> % .-target .-value)]) + :value (or @(rf/subscribe [:ddb]) "")} + (for [db @(rf/subscribe [:ddbs])] + [:option {:key db :value db} db])]] + :header-extra [:span.float-right2.h4 @(rf/subscribe [:ddb])]]) (defn wick-card [] diff --git a/src/cljs/org/parkerici/enflame/views.cljs b/src/cljs/org/parkerici/enflame/views.cljs index 2980e32..5649ae3 100644 --- a/src/cljs/org/parkerici/enflame/views.cljs +++ b/src/cljs/org/parkerici/enflame/views.cljs @@ -173,8 +173,11 @@ (defn error [[status response]] - [:div {:style {:color "red"}} - (str status ": " response)]) + [:div + [:h3 "Error"] + [:pre {:style {:color "red"}} + (with-out-str (pprint/pprint status))] + ]) (defn download-link [] From 5c1f1993bdf117f67f069998567caa6cb073665e Mon Sep 17 00:00:00 2001 From: Mike Travers Date: Wed, 1 Mar 2023 14:41:05 -0800 Subject: [PATCH 05/19] Devops to deploy on AWS App Runner --- Dockerfile | 10 + README-devops.org | 17 ++ bin/build.sh | 34 +++ resources/opencandel-config.edn | 4 +- src/clj/org/parkerici/enflame/sparql.clj | 295 -------------------- src/clj/org/parkerici/enflame/uniprot.clj | 314 ---------------------- 6 files changed, 63 insertions(+), 611 deletions(-) create mode 100644 Dockerfile create mode 100644 README-devops.org create mode 100755 bin/build.sh delete mode 100644 src/clj/org/parkerici/enflame/sparql.clj delete mode 100644 src/clj/org/parkerici/enflame/uniprot.clj diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ce31ad7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM openjdk:13 +MAINTAINER Mike Travers "mt@alum.mit.edu" + +ENV PORT 1991 +EXPOSE 1991 + +ADD target ~/target +WORKDIR ~/target + +ENTRYPOINT ["java", "-jar", "enflame-standalone.jar", "1991", "resources/opencandel-config.edn"] diff --git a/README-devops.org b/README-devops.org new file mode 100644 index 0000000..5c60030 --- /dev/null +++ b/README-devops.org @@ -0,0 +1,17 @@ +* Scope of this document + + + +* Configuration + +* Deployment + +* AWS Services + +** Datomic Cloud + +** App Runner + +** ECR (Container Registry) + +** TODO storage service for library diff --git a/bin/build.sh b/bin/build.sh new file mode 100755 index 0000000..7bbd49e --- /dev/null +++ b/bin/build.sh @@ -0,0 +1,34 @@ +# Build Docker image + +# TODO parameterize properly + +export VERSION=$(git rev-parse --short HEAD) +echo Building version $VERSION + +# get latest version of schema (TODO: is this a good idea?) +# doesn't seem to actually work +# git submodule update --init --recursive + +# TODO needs rethinking +# Run Alzabo to build schemas +# cd alzabo; lein with-profile prod do clean, run documentation candel "*", cljsbuild once; cd .. + +lein uberjar + +# TODO abort if lein fails, duh. + +# Build Docker image + +docker build -t cbio . + +# verify +# docker run -p 8080 cbio + + +docker tag cbio:latest 733151965047.dkr.ecr.us-east-1.amazonaws.com/cbio:latest + +# Upload to AWS repository + +aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 733151965047.dkr.ecr.us-east-1.amazonaws.com + +docker push 733151965047.dkr.ecr.us-east-1.amazonaws.com/cbio:latest diff --git a/resources/opencandel-config.edn b/resources/opencandel-config.edn index 52659ab..59ebbf2 100644 --- a/resources/opencandel-config.edn +++ b/resources/opencandel-config.edn @@ -1,9 +1,9 @@ {:source {:type :datomic-cloud ;The data source. :config {:server-type :ion ;Passed directly to Datomic Client API :region "us-east-1" ;; e.g. us-east-1 - :system "PublicCANDEL" + :system "PublicCANDEL5" ;; :creds-profile "" - :endpoint "https://pepaga6o2e.execute-api.us-east-1.amazonaws.com/"} + :endpoint "https://nazpex6ueb.execute-api.us-east-1.amazonaws.com"} } :schema "resources/candel-schema-1-3-1.edn" ;filename of an Alzabo schema diff --git a/src/clj/org/parkerici/enflame/sparql.clj b/src/clj/org/parkerici/enflame/sparql.clj deleted file mode 100644 index 307338e..0000000 --- a/src/clj/org/parkerici/enflame/sparql.clj +++ /dev/null @@ -1,295 +0,0 @@ -(ns org.parkerici.enflame.sparql - (:require - [clj-http.client :as client] - [arachne.aristotle.query :as q] - [arachne.aristotle.registry :as reg] - [arachne.aristotle.graph :as graph] - [clojure.data.json :as json] - [org.parkerici.multitool.core :as u] - ) - (:import (org.apache.jena.rdf.model Model Resource Literal Property Statement RDFNode)) - ) - -;;; Not clear what Aristotle does that isn't better handled by Jena SSE https://jena.apache.org/documentation/notes/sse.html - - -(defn ->sparql [bgp & {:keys [limit]}] - (let [query (-> bgp - q/build - org.apache.jena.sparql.algebra.OpAsQuery/asQuery)] - ;; TODO nany other options, try (bean query) - (when limit - (.setLimit query limit)) - (str query))) - -(comment - (reg/prefix 'ds "https://data.lacity.org/resource/zzzz-zzzz/") - (->sparql '(:bgp {:rdf/about ?e, :ds/zip_code "90001", :ds/total_population ?pop}))) - -;;; This could probably be improved -;;; see https://github.com/ajoberstar/ike.cljj -(defn make-consumer [collector] - (reify java.util.function.Consumer - (accept [& thing] - (swap! collector conj thing)))) - -;;; Note: not a Graph -(defn sparql-source - [url] - (-> (org.apache.jena.rdfconnection.RDFConnectionRemote/create) - (.destination url))) - -;;; TODO - want to try q/run, but it requires a Graph object and I can't figure it out... - -(defn do-query-raw - "Source is an org.apache.jena.rdfconnection.RDFConnectionRemoteBuilder - q is a sparql string" - [source q] - (let [collector (atom [])] - (.querySelect (.build source) q - (make-consumer collector)) - collector)) - -(defn result-binding->clj [rb] - (let [bindings (.getBinding rb) - vars (iterator-seq (.vars bindings))] - (zipmap (map #(keyword (.getVarName %)) vars) - (map #(graph/data (.get bindings %)) vars)))) - -(defn do-query [source q] - (let [results @(do-query-raw source q)] - (map result-binding->clj results))) - - -(defn xquery [& args] - (apply q/run args)) - - -;;; Reshaping tools - - -(defn entify-1 - [query-results] - (u/map-values (fn [c] (map :o c)) - (group-by :p query-results))) - - -(defn entify - "Turn a query results with :s :p :o fields into a set of maps" - [query-results] - (->> query-results - (group-by :s) - (u/map-values entify-1))) - -;;; Searching for usable endpoints - -(def sparql-dbs (clojure.data.json/read-str (slurp "/Users/mtravers/Downloads/query (1).json") - :key-fn keyword)) - - -(defn check-link - [url] - (client/head url - {:cookie-policy :standard - :trace-redirects true - :redirect-strategy :graceful})) - -(defn test-endpoint - [e] - (prn :testing e) - (check-link (:u e))) - -;;; From goddinpotty -(defn check-external-links - "Find bad external links in the background." - [] - (let [bads (atom []) - goods (atom [])] - (doseq [{:keys [u item] :as db} sparql-dbs] - (future-call - #(try - (prn :checking db) - (swap! goods conj [db (check-link u)]) - (catch Throwable e (swap! bads conj [db e]))))) - [bads goods])) - - -(comment -;;; These are some that at least respond. -("http://vulcan.cs.uga.edu/sparql" ;;; Prokino, see http://prokino.uga.edu/prokino/query/sparql - - - "http://opencitations.net/sparql" - "https://sparql.proconsortium.org/virtuoso/sparql" - "http://opendatacommunities.org/sparql" - "http://data.allie.dbcls.jp/sparql" - "https://bgee.org/sparql" - "https://ld.cultural.jp/sparql" - "http://genome.microbedb.jp/sparql" - "https://data.europa.eu/euodp/sparqlep" - "http://id.nlm.nih.gov/mesh/sparql" - "https://colil.dbcls.jp/sparql" - "http://data.archiveshub.ac.uk/sparql" - "http://lod.nl.go.kr/sparql" - "https://query.inventaire.io" - "https://sparql.uniprot.org/sparql" - "http://www.europeandataportal.eu/sparql" - "https://data.norge.no/sparql" - "http://id.sgcb.mcu.es/sparql" - "http://data.bibliotheken.nl/sparql" - "http://rdf.disgenet.org/sparql/" - "http://statistics.data.gov.uk/sparql" - "http://statistics.data.gov.uk/sparql" - "http://linkedgeodata.org/sparql" - "https://id.ndl.go.jp/auth/ndla/sparql" - "http://cultura.linkeddata.es/sparql" - "https://jpsearch.go.jp/rdf/sparql/" - "https://isidore.science/sqe" - "https://commons-query.wikimedia.org/" - "https://mediag.bunka.go.jp/madb_lab/lod/sparql/" - "http://patho.phenomebrowser.net/sparql/" - "http://sparql.archives-ouvertes.fr/sparql" - "https://www.dictionnairedesfrancophones.org/sparql" - "https://database.factgrid.de/query/" - "https://datos.gob.es/es/sparql" - "https://sparql.orthodb.org/sparql" - "https://data.gov.cz/sparql" - "http://data.archaeologydataservice.ac.uk/sparql/repositories/archives" - "https://lingualibre.org/bigdata/namespace/wdq/sparql" - "https://slod.fiz-karlsruhe.de/sparql" - "https://druid.datalegend.net/AdamNet/Heritage/sparql/Heritage" - "http://www.genome.jp/sparql/linkdb" - "http://ma-graph.org/sparql" - "http://sparql.wikipathways.org/" - "http://bnb.data.bl.uk/sparql" - "http://data.culture.fr/thesaurus/sparql" - "https://data.muziekweb.nl/MuziekwebOrganization/Muziekweb/sparql/Muziekweb" - "http://data.bnf.fr/sparql/" - "https://labs.onb.ac.at/en/tool/sparql/" - "http://dati.camera.it/sparql" - "https://tora.entryscape.net/snorql" - "https://sparql.rhea-db.org/sparql" - "http://data.nobelprize.org/" - "https://triplestore.iccu.sbn.it/sparql" - "https://triplestore.iccu.sbn.it/sparql" - "https://query.linkedopendata.eu/sparql" - "http://bio2rdf.org/sparql" - "http://vocabulary.curriculum.edu.au/PoolParty/sparql/scot" - "http://data.cervantesvirtual.com/openrdf-sesame/repositories/data" - "https://libris.kb.se/sparql" - "http://data.nationallibrary.fi/bib/sparql" - "http://data.persee.fr/sparql" - "https://data.idref.fr/sparql" - "https://data.cssz.cz/sparql" - "https://www.orpha.net/sparql" - "https://www.orpha.net/sparql" - "https://datos-abertos.galiciana.gal/pt/sparql" - "https://xn--slovnk-7va.gov.cz/sparql" - "https://idsm.elixir-czech.cz/sparql/" - "http://dbpedia.org/sparql" - "http://lod.openaire.eu/endpoint" - "http://sparql.europeana.eu/" - "http://data.ordnancesurvey.co.uk/datasets/os-linked-data/apis/sparql" - "https://query.wikidata.org/sparql") -) - - -;;; Abstracted from Uniprot -(defn ^:api q - [endpoint sparql] - (if (string? sparql) - (do-query (sparql-source endpoint) sparql) - (q endpoint (->sparql sparql)))) - - -(defn ontology-query - [endpoint] - (-> - (entify - (q endpoint - '(:bgp [?s :rdfs/isDefinedBy ?uniprot] - [?s ?p ?o]))))) - -(defn ontology-query-2 - [endpoint] - (-> - (entify - (q endpoint - '(:bgp [?s :rdf/type :owl/Class] - [?s ?p ?o]))))) - -(defn ^:api instances - "Does a pull of all instances of class" - [endpoint class] - (entify - (q endpoint - `(:bgp [?s :rdf/type ~class] - [?s ?p ?o])))) - -(defn ^:api instances-only - "Gets all instances of class" - [endpoint class] - (q endpoint - `(:bgp [?s :rdf/type ~class] - ))) - -(defn ^:api pull - [endpoint ent] - (entify-1 - (q endpoint - `(:bgp [~ent ?p ?o])) - )) - -(defn ^:api pull2 - [endpoint ent] - (entify-1 - (q endpoint - `(:conditional - (:bgp [?o ?p2 ?o2]) - (:bgp [~ent ?p ?o]))))) - - -(defn ^:api pull2 - [endpoint ent] - (q endpoint - `(:bgp [~ent ?p ?o] - [?o ?p2 ?o2]))) - -(defn ^:api pull3 - [endpoint ent] - (q endpoint - `(:bgp [~ent ?p ?o] - [?o ?p2 ?o2] - [?o2 ?p3 ?o3] - ))) - - - - -(defn ^:api antipull - [endpoint ent] - (entify-1 - (q endpoint - `(:bgp [?o ?p ~ent])) - )) - -(defn attributes - [endpoint att] - (q endpoint - `(:bgp [?s ~att ?o])) - ) - - -(comment -;;; Look here for examples -;;; https://github.com/arachne-framework/aristotle/blob/master/test/arachne/aristotle/query_test.clj#L90 -;;; This runs but seems to ignore the aggregate spec -(q '(:group (?s) ((?s (count ?s))) (:bgp [?s :rdfs/domain :prokino/LigandActivity]))) - - - -;;; Example of DISTINCT -(def x (sq/q endpoint - '(:distinct (:project [?o] (:bgp [?s :rdf/type :prokino/LigandActivity] - [?s :prokino/hasMOA ?o]))))) -) diff --git a/src/clj/org/parkerici/enflame/uniprot.clj b/src/clj/org/parkerici/enflame/uniprot.clj deleted file mode 100644 index ad4a58d..0000000 --- a/src/clj/org/parkerici/enflame/uniprot.clj +++ /dev/null @@ -1,314 +0,0 @@ -(ns org.parkerici.enflame.uniprot - (:require - [org.parkerici.enflame.sparql :as sq] - [arachne.aristotle.registry :as reg] - [org.parkerici.multitool.core :as u] - [org.parkerici.multitool.cljcore :as ju] - [clojure.set :as set] - ) ) - -;;; These are silly -(reg/prefix 'uniprot "http://purl.uniprot.org/core/") -(reg/prefix 'unipath "http://purl.uniprot.org/unipathway/") -(reg/prefix 'unicite "http://purl.uniprot.org/citations/") -(reg/prefix 'unidb "http://purl.uniprot.org/database") -(reg/prefix 'dcterms "http://purl.org/dc/terms/") -(reg/prefix 'unienzyme "http://purl.uniprot.org/enzyme/") -(reg/prefix 'skos "http://www.w3.org/2004/02/skos/core#") - -;;; Try (this does not seem to work, sigh) -(reg/prefix 'uniuni "http://purl.uniprot.org/") - -(defn uniprot-q - [sparql] - (if (string? sparql) - (sq/do-query (sq/sparql-source "https://sparql.uniprot.org/") sparql) - (uniprot-q (sq/->sparql sparql :limit 100000)))) ;TODO limit temp :limit 1000 - - -(def external-ontology - '{:rdf/Statement {:rdf/type (:owl/Class)}}) - -;;; TODO shouldn't run on compile -(defonce uniprot-ontology - (-> - (sq/entify - (uniprot-q - '(:bgp [?s :rdfs/isDefinedBy ?uniprot] - [?s ?p ?o]))) - (merge external-ontology) - ;; This one field comes back with an unserializable object, just patch it - ;; Real thing (.-lexicalValue _) if need be - (assoc-in [:uniprot/Pathway :rdfs/label] '("Pathway")))) - -;;; TODO damn I wish these were more composable - - -;;; Note :unipath/399.28.3.3 doesn't work with Clojure reader, argh -#_ -(pull (keyword "unipath" "399.28.3.3")) - - -(defn filtered-by - [field value] - (u/dissoc-if (fn [[n d]] - (not (some #(= value %) (field d)))) - uniprot-ontology)) - -(defn filtered-by-any - [field values] - (u/dissoc-if (fn [[n d]] - (empty? (set/intersection (set (field d)) (set values)))) - uniprot-ontology)) - - -(defn filtered-by-rdf-type - [type] - (u/dissoc-if (fn [[n d]] - ;; TODO assuming a single type - (not (= (:rdf/type d) (list type)))) - uniprot-ontology) ) - -(defn classes - [] - (filtered-by-rdf-type :owl/Class)) - -(defn subclasses - [class] - (filtered-by :rdfs/subClassOf class)) - -(def all-subclasses - (u/transitive-closure (comp keys subclasses))) - -(defn properties - [] - (merge (filtered-by-rdf-type :owl/DatatypeProperty) - (filtered-by-rdf-type :owl/ObjectProperty))) - -(defn properties-for-domain - [class] - (filtered-by-any :rdfs/domain (all-subclasses class))) - - - - - - - -(defn uniprot? - [ent] - (and (keyword? ent) - (= "uniprot" (namespace ent)))) - -(defn top-classes - [] - (filter (fn [[c d]] - (not (contains? (classes) (:rdfs/subClassOf d)))) - (classes))) - -(defn top-classes - [] - (let [non-tops (keys (filtered-by-any :rdfs/subClassOf (keys (classes))))] - (apply dissoc (classes) non-tops))) - -#_ -(:uniprot/Database - :uniprot/Structured_Name - :uniprot/Enzyme_Regulation_Annotation - :uniprot/Enzyme - :uniprot/Excluded_Proteome - :uniprot/Gene - :uniprot/Citation - :uniprot/Attribution - :uniprot/Organelle - :uniprot/Status - :uniprot/Part - :uniprot/Participant - :uniprot/Journal - :uniprot/Structure_Mapping_Statement - :uniprot/Proteome - :uniprot/Nucleotide_Mapping_Statement - :uniprot/Method - :uniprot/Taxon - :uniprot/Molecule - :uniprot/Obsolete - :uniprot/Disease - :uniprot/Resource - :uniprot/Proteome_Component - :uniprot/Cluster - :uniprot/Domain_Assignment_Statement - :uniprot/Protein_Existence - :uniprot/Subcellular_Location - :uniprot/Transposon - :uniprot/Plasmid - :uniprot/Concept - :uniprot/Annotation - :uniprot/Endpoint_Statement - :uniprot/Protein - :uniprot/Tissue - :uniprot/Sequence - :uniprot/Strain - :uniprot/Interaction - :uniprot/Catalytic_Activity - :uniprot/Not_Obsolete - :uniprot/Pathway - :uniprot/Citation_Statement - :uniprot/Rank - :uniprot/Reviewed) - -(count (instances :uniprot/Pathway)) -3117 -(count (instances :uniprot/Disease)) -6202 -(count (instances :uniprot/Molecule)) -0 ;; uh uo - - -(defn describe - [ent] - (concat - (uniprot-q `(:bgp [~ent ?p ?o])) - (uniprot-q `(:bgp [?s ?p ~ent])))) - - -(comment -(frequencies (map :rdf/type (vals uniprot-ontology) )) -{(:owl/DatatypeProperty) 43, - (:owl/ObjectProperty) 67, - (:owl/NamedIndividual :owl/Thing :uniprot/Organelle) 9, - (:owl/NamedIndividual :owl/Thing :uniprot/Rank) 31, - (:owl/Class) 168, - (:owl/InverseFunctionalProperty :owl/FunctionalProperty :owl/ObjectProperty) 1, - (:owl/FunctionalProperty :owl/ObjectProperty) 6, - (:owl/FunctionalProperty :owl/DatatypeProperty) 31, - (:owl/NamedIndividual :owl/Thing :uniprot/Status) 4, - (:owl/NamedIndividual :owl/Thing :uniprot/Protein_Existence) 5, - (:owl/NamedIndividual :owl/Thing :uniprot/Mass_Measurement_Method) 7, - (:owl/NamedIndividual :owl/Thing :uniprot/Structure_Determination_Method) 7} -) - - - -#_ -(frequencies (map :rdfs/domain (vals (properties)))) - -;;; Huh. -#_ -{nil 17, - (:uniprot/Subcellular_Location_Annotation) 1, - (:uniprot/Structured_Name) 1, - (:uniprot/Proteome) 2, - (:uniprot/Interaction) 1, - (:uniprot/Reviewed_Protein) 1, - (_626915beb033654fc13c8409d68fbefb) 1, - (:uniprot/RNA_Editing_Annotation) 1, - (:uniprot/External_Sequence) 1, - (_135e6ace0ba508ab2319c50063cc0ede) 1, - (_7a99d05307375d434d4a3f01c938cad7) 1, - (_b868ac3eb7960429cb539cea5f6300ae) 1, - (:uniprot/Gene) 2, - (_ade505809c0086211dc02c0f9464258e) 1, - (:uniprot/Resource) 1, - (_cfd54a0e1d32a8d62b76f3490d7f2311) 1, - (:uniprot/Transcript_Resource) 2, - (:uniprot/Published_Citation) 2, - (_e978c46c81fd658d3e99796c5eaf4502) 1, - (_5fcb391e3136fc76ba988a8b5f961505) 1, - (_941b9dd17dd300caedb9cb8c0e3f958e) 1, - (:uniprot/Disease_Annotation) 1, - (_d0e6352838bda96a5d57075fed61b8fc) 1, - (:uniprot/Simple_Sequence) 2, - (:uniprot/Protein) 13, - (:uniprot/Modified_Sequence) 1, - (:uniprot/Enzyme) 2, - (:uniprot/Catalytic_Activity_Annotation) 2, - (_49396fea4d19b3d5b39285cc1252056b) 1, - (_e2daa4cde5ccb48ceaa946a7c97ec83e) 1, - (:uniprot/Journal) 1, - (_3725fc1a96109bd014937233e0cc1e80) 1, - (:uniprot/Cluster) 2, - (_086cb0592bb819a0cd93d44d3bf577d8) 1, - (:uniprot/Binding_Site_Annotation) 2, - (_63f5f671dd82a005a9e91a5c22c458a5) 1, - (:uniprot/Structure_Mapping_Statement) 1, - (_a2745b16016d308213a790963118d9a8) 1, - (:rdfs/Resource) 1, - (:uniprot/Thesis_Citation) 1, - (_d84a5a698f10400d9ffba7376653fc21) 1, - (:uniprot/Nucleotide_Resource) 2, - (_9d3897b13bb29c76ec292b254542decb) 1, - (:uniprot/Subcellular_Location) 1, - (_612e2c86654093537c55009db223f480) 1, - (:uniprot/Book_Citation) 2, - (:uniprot/Kinetics_Annotation) 2, - (:uniprot/Sequence) 3, - (:uniprot/Cofactor_Annotation) 1, - (:uniprot/Catalytic_Activity) 1, - (:uniprot/Citation) 5, - (:uniprot/Database) 5, - (:uniprot/Submission_Citation) 1, - (:uniprot/Taxon) 4, - (:uniprot/Attribution) 1, - (:uniprot/Citation_Statement) 2} - - -;;; Alzabo schema gen - -;;; TODO should include the real URI somewhere - -;;; TODO Alzabo has no concept of subclass, would be interesting to add -;;; For now, it compresses everything into top classes - -;;; Remove namespace (see u/d-ns) -(defn nons - [key] - (if (keyword key) - (keyword (name key)) - key)) - -;;; TODO add skos etc fields -(defn class-alzabo-fields - [class] - (apply - merge - (for [[n d] (properties-for-domain class)] - {(nons n) - {:type (or (nons (first (:rdfs/range d))) - :string) ;temp but nil doen't work - ;; :cardinality ... - :uri n - :attribute n ;aka :uri, but this leverages existing mechanisms - :doc (first (:rdfs/comment d))}} - ))) - -(defn alzabo - [] - (u/clean-walk - {:title "UNIPROT" - :kinds - (apply - merge - (for [[tc tc-def] (top-classes)] - {(nons tc) - {:doc (first (:rdfs/comment tc-def)) - :title (first (:rdfs/label tc-def)) ;not actually used or defined - :fields (or (class-alzabo-fields tc) {}) - :uri tc - }}))} - nil?)) - -#_ -(ju/schppit "uniprot-ontology.edn" uniprot-ontology) - -#_ -(ju/schppit "resources/uniprot-alzabo.edn" (alzabo)) - - -;;; Regex usage - - - - `(:bgp [?protein :rdf/type :uniprot/Protein] - [?protein :uniprot/classifiedWith ?concept] - [?concept :rdfs/label ?clabel] - [(regex ?clabel "FOO.*" "")]) From 99baff1f94cc31fe05c046943f0ad2dd61962e4d Mon Sep 17 00:00:00 2001 From: Mike Travers Date: Wed, 1 Mar 2023 17:17:47 -0800 Subject: [PATCH 06/19] Tweaks to deployment, this is actually working on an AWS server --- Dockerfile | 6 ++--- project.clj | 1 + resources/opencandel-config.edn | 10 ++++---- src/clj/org/parkerici/enflame/admin.clj | 6 +++-- src/clj/org/parkerici/enflame/config.clj | 4 ++- src/clj/org/parkerici/enflame/gcs.clj | 25 ++++++++++--------- .../org/parkerici/enflame/library/core.clj | 19 ++++++++++++++ src/clj/org/parkerici/enflame/server.clj | 6 ++--- .../org/parkerici/enflame/library/item.cljc | 2 +- 9 files changed, 52 insertions(+), 27 deletions(-) create mode 100644 src/clj/org/parkerici/enflame/library/core.clj diff --git a/Dockerfile b/Dockerfile index ce31ad7..2349600 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ FROM openjdk:13 MAINTAINER Mike Travers "mt@alum.mit.edu" -ENV PORT 1991 -EXPOSE 1991 +ENV PORT 1996 +EXPOSE 1996 ADD target ~/target WORKDIR ~/target -ENTRYPOINT ["java", "-jar", "enflame-standalone.jar", "1991", "resources/opencandel-config.edn"] +ENTRYPOINT ["java", "-jar", "enflame-standalone.jar", "opencandel-config.edn"] diff --git a/project.clj b/project.clj index 086ec12..6e66b19 100644 --- a/project.clj +++ b/project.clj @@ -91,6 +91,7 @@ :css-dirs ["resources/public/css"]} :uberjar-name "enflame-standalone.jar" + :resource-paths ["resources"] :profiles {:dev diff --git a/resources/opencandel-config.edn b/resources/opencandel-config.edn index 59ebbf2..e2f70c4 100644 --- a/resources/opencandel-config.edn +++ b/resources/opencandel-config.edn @@ -6,10 +6,9 @@ :endpoint "https://nazpex6ueb.execute-api.us-east-1.amazonaws.com"} } - :schema "resources/candel-schema-1-3-1.edn" ;filename of an Alzabo schema + :schema "candel-schema-1-3-1.edn" ;filename of an Alzabo schema - :library {:gcs-project :redacted ;GCS Datastore project for storiing library entries - :gcs-console-url :redacted} + :library {} ;TODO need an AWS library solution :query-generator :candel-generate ;The query generator @@ -17,10 +16,11 @@ :query ;; I guess no point to this until Wick server is brought up ;; :candel/wick - :share + ;; Library also temporarily out of service + ;; :share :compact ;debug only :browser ] - :port 1997 ;Port on which to serve application + :port 1996 ;Port on which to serve application :dev? true ;debug only } diff --git a/src/clj/org/parkerici/enflame/admin.clj b/src/clj/org/parkerici/enflame/admin.clj index 766a693..8aff365 100644 --- a/src/clj/org/parkerici/enflame/admin.clj +++ b/src/clj/org/parkerici/enflame/admin.clj @@ -3,6 +3,7 @@ [org.parkerici.enflame.html :as html] [hiccup.util :as hu] [clojure.java.shell :as sh] + [org.parkerici.multitool.core :as u] )) (defn map-table @@ -17,8 +18,9 @@ (hu/escape-html (str (get map key))))]])]]) (defn git-info [] - {:commit (:out (sh/sh "git" "log" "-1" "--format=short")) - :branch (:out (sh/sh "git" "rev-parse" "--abbrev-ref" "HEAD"))}) + (u/ignore-errors + {:commit (:out (sh/sh "git" "log" "-1" "--format=short")) + :branch (:out (sh/sh "git" "rev-parse" "--abbrev-ref" "HEAD"))})) (defn view [req] diff --git a/src/clj/org/parkerici/enflame/config.clj b/src/clj/org/parkerici/enflame/config.clj index c02e421..2607601 100644 --- a/src/clj/org/parkerici/enflame/config.clj +++ b/src/clj/org/parkerici/enflame/config.clj @@ -3,6 +3,7 @@ [clojure.edn :as edn] [clojure.pprint :as pprint] [org.parkerici.multitool.core :as u] + [clojure.java.io :as io] )) ;;; TODO Aero-ize @@ -12,7 +13,7 @@ (defn load-config [file] (let [env-config {} ; if we want to supply some config from environment - file-config (edn/read-string (slurp file)) + file-config (edn/read-string (slurp (io/resource file))) config (merge env-config file-config)] (pprint/pprint config) (reset! the-config config))) @@ -25,6 +26,7 @@ ;;; TODO use version (u/defn-memoized read-schema [version] (-> (config :schema) + io/resource slurp edn/read-string)) diff --git a/src/clj/org/parkerici/enflame/gcs.clj b/src/clj/org/parkerici/enflame/gcs.clj index 1855577..e272ad1 100644 --- a/src/clj/org/parkerici/enflame/gcs.clj +++ b/src/clj/org/parkerici/enflame/gcs.clj @@ -8,25 +8,26 @@ ;;; Interface to GCS Datastore service -(def service +;;; TODO this is very bad. A little less bad made lazy +(u/def-lazy service (.getService (-> (DatastoreOptions/newBuilder) (.setProjectId (config/config :library :gcs-project)) (.build)))) (u/defn-memoized key-factory [kind] - (-> (.newKeyFactory service) + (-> (.newKeyFactory @service) (.setKind kind))) (defn get-entity-key [kind ent-id] - (-> (.newKeyFactory service) + (-> (.newKeyFactory @service) (.setKind kind) (.newKey ent-id))) (defn delete-entity [ent-id] - (.delete service (into-array [(get-entity-key ent-id)]))) + (.delete @service (into-array [(get-entity-key ent-id)]))) (defn to-map [ent] @@ -40,7 +41,7 @@ [kind ent-id] (let [k (get-entity-key kind ent-id)] (when k - (to-map (.get service k))))) + (to-map (.get @service k))))) ;;: TODO parameterize query maybe (defn list-items @@ -49,7 +50,7 @@ (.setKind kind))] (map to-map (iterator-seq - (.run service (.build query)))))) + (.run @service (.build query)))))) (defn latest-item [kind field] @@ -60,7 +61,7 @@ (to-map (first (iterator-seq - (.run service (.build query))))))) + (.run @service (.build query))))))) (defn all-items [kind] @@ -69,7 +70,7 @@ )] (map to-map (iterator-seq - (.run service (.build query)))))) + (.run @service (.build query)))))) ;;; Not presently called (defn upload-property-names @@ -77,7 +78,7 @@ (let [key-factory (key-factory "PropertyName") v (map name (keys fields)) entities (map #(.build (Entity/newBuilder (.newKey key-factory %))) v)] - (.put service (into-array entities)))) + (.put @service (into-array entities)))) (defn big-string [s] (-> s @@ -98,10 +99,10 @@ (defn upload [kind item big-keys] (let [key (->> (.newKey (key-factory kind)) - (.allocateId service)) + (.allocateId @service)) builder (-> (Entity/newBuilder key) (add-fields-to-entity item big-keys))] - (.add service (.build builder)))) + (.add @service (.build builder)))) ;;; Datastore does not support OR or IN operatore, so we fake it ;;; Warning: this does a crossproduct on all multiple-valued prop filters, can be expensive @@ -117,4 +118,4 @@ (when property-filters (.setFilter query property-filters)) (into [] (iterator-seq - (.run service (.build query)))))) + (.run @service (.build query)))))) diff --git a/src/clj/org/parkerici/enflame/library/core.clj b/src/clj/org/parkerici/enflame/library/core.clj new file mode 100644 index 0000000..00ad443 --- /dev/null +++ b/src/clj/org/parkerici/enflame/library/core.clj @@ -0,0 +1,19 @@ +(ns org.parkerici.enflame.library.core + (:require [org.parkerici.enflame.gcs :as gcs] + [org.parkerici.enflame.config :as config] + ) + ) + +;;; GCS implementation + +(def big-keys #{::blockdef ::image}) + +(defn upload + [item] + (case (config/config [:library :type]) + :gcs (gcs/upload "EnflameItem" item big-keys))) + +(defn get-item + [key] + (case (config/config [:library :type]) + :gcs (gcs/get-item "EnflameItem" (Long. key)))) diff --git a/src/clj/org/parkerici/enflame/server.clj b/src/clj/org/parkerici/enflame/server.clj index 8d1b254..3edb1c9 100644 --- a/src/clj/org/parkerici/enflame/server.clj +++ b/src/clj/org/parkerici/enflame/server.clj @@ -7,7 +7,7 @@ [org.parkerici.enflame.schema :as schema] [org.parkerici.enflame.config :as config] [org.parkerici.enflame.oauth :as oauth] - [org.parkerici.enflame.gcs :as gcs] + [org.parkerici.enflame.library.core :as lib] [org.parkerici.enflame.library.item :as item] [org.parkerici.enflame.library.view :as library-view] [org.parkerici.multitool.core :as u] @@ -63,13 +63,13 @@ (defn handle-save [req] (let [item (get-in req [:params :item]) - response (gcs/upload "EnflameItem" item item/big-keys)] + response (lib/upload item)] (response/response ;response^4 {:response response}))) (defn handle-get [key] - (let [item (item/localize-item (gcs/get-item "EnflameItem" (Long. key)))] + (let [item (item/localize-item (lib/get-item key))] ;; TODO handle not found (response/response item))) diff --git a/src/cljc/org/parkerici/enflame/library/item.cljc b/src/cljc/org/parkerici/enflame/library/item.cljc index f63af8d..aee86e4 100644 --- a/src/cljc/org/parkerici/enflame/library/item.cljc +++ b/src/cljc/org/parkerici/enflame/library/item.cljc @@ -24,7 +24,7 @@ [item] (or (::label item) (::description item) (::query item))) -(def big-keys #{::blockdef ::image}) +#_(def big-keys #{::blockdef ::image}) ;;; Convert from gcs From d9b2d6754b24ae4c7f943b1b9c51ed4f66e7506f Mon Sep 17 00:00:00 2001 From: Mike Travers Date: Thu, 2 Mar 2023 10:29:29 -0800 Subject: [PATCH 07/19] fix icons, hide compacted pane --- README-devops.org | 9 +++++++++ resources/opencandel-config.edn | 2 +- resources/public/index.html | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/README-devops.org b/README-devops.org index 5c60030..7c705ed 100644 --- a/README-devops.org +++ b/README-devops.org @@ -6,12 +6,21 @@ * Deployment +Once App Runner is set up, pushing a new Docker image will restart the server. + +bin/build.sh does all the necessary steps + + * AWS Services +These were all cobbled together by hand, I suppose the TODO is to make a CloudFormation them. + ** Datomic Cloud ** App Runner ** ECR (Container Registry) +** IAM + ** TODO storage service for library diff --git a/resources/opencandel-config.edn b/resources/opencandel-config.edn index e2f70c4..3964fdd 100644 --- a/resources/opencandel-config.edn +++ b/resources/opencandel-config.edn @@ -18,7 +18,7 @@ ;; :candel/wick ;; Library also temporarily out of service ;; :share - :compact ;debug only + ;; :compact ;debug only :browser ] :port 1996 ;Port on which to serve application diff --git a/resources/public/index.html b/resources/public/index.html index 0fcc0cb..e861979 100644 --- a/resources/public/index.html +++ b/resources/public/index.html @@ -3,7 +3,7 @@ Enflame query builder - + From c451d99a5c34adf80e843b88963e92e4bc71857d Mon Sep 17 00:00:00 2001 From: Mike Travers Date: Fri, 3 Mar 2023 08:08:12 -0800 Subject: [PATCH 08/19] fix documentation --- {doc => bin}/build-guide.sh | 0 bin/build.sh | 2 ++ resources/public/doc | 1 + 3 files changed, 3 insertions(+) rename {doc => bin}/build-guide.sh (100%) create mode 120000 resources/public/doc diff --git a/doc/build-guide.sh b/bin/build-guide.sh similarity index 100% rename from doc/build-guide.sh rename to bin/build-guide.sh diff --git a/bin/build.sh b/bin/build.sh index 7bbd49e..556ff48 100755 --- a/bin/build.sh +++ b/bin/build.sh @@ -5,6 +5,8 @@ export VERSION=$(git rev-parse --short HEAD) echo Building version $VERSION +bin/build-doc.sh + # get latest version of schema (TODO: is this a good idea?) # doesn't seem to actually work # git submodule update --init --recursive diff --git a/resources/public/doc b/resources/public/doc new file mode 120000 index 0000000..7e57b0f --- /dev/null +++ b/resources/public/doc @@ -0,0 +1 @@ +../../doc \ No newline at end of file From 38439f48c84571acf8706d139f97ac4fdf16910a Mon Sep 17 00:00:00 2001 From: Mike Travers Date: Sat, 4 Mar 2023 17:08:43 -0800 Subject: [PATCH 09/19] fix schema --- bin/build.sh | 2 +- resources/public/schema | 1 + src/cljs/org/parkerici/enflame/core.cljs | 1 - src/cljs/org/parkerici/enflame/views.cljs | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) create mode 120000 resources/public/schema diff --git a/bin/build.sh b/bin/build.sh index 556ff48..70b5542 100755 --- a/bin/build.sh +++ b/bin/build.sh @@ -5,7 +5,7 @@ export VERSION=$(git rev-parse --short HEAD) echo Building version $VERSION -bin/build-doc.sh +bin/build-guide.sh # get latest version of schema (TODO: is this a good idea?) # doesn't seem to actually work diff --git a/resources/public/schema b/resources/public/schema new file mode 120000 index 0000000..fa2ecdd --- /dev/null +++ b/resources/public/schema @@ -0,0 +1 @@ +../../../alzabo/resources/public/schema/1.3.1 \ No newline at end of file diff --git a/src/cljs/org/parkerici/enflame/core.cljs b/src/cljs/org/parkerici/enflame/core.cljs index 88a1b06..07ff315 100644 --- a/src/cljs/org/parkerici/enflame/core.cljs +++ b/src/cljs/org/parkerici/enflame/core.cljs @@ -1,4 +1,3 @@ - (ns org.parkerici.enflame.core (:require [reagent.dom :as rdom] diff --git a/src/cljs/org/parkerici/enflame/views.cljs b/src/cljs/org/parkerici/enflame/views.cljs index 5649ae3..fd93c72 100644 --- a/src/cljs/org/parkerici/enflame/views.cljs +++ b/src/cljs/org/parkerici/enflame/views.cljs @@ -336,7 +336,7 @@ (toplink [:span [:b "Enflame "] [:img {:src "favicon.ico" :width "24px"}]] "http://github.com/ParkerICI/enflame") (toplink "Tutorial" "/doc/tutorial.html") (toplink "Doc" "/doc/guide.html") - (toplink "Schema" (str "/schema/" "" "/index.html")) ;TODO version + (toplink "Schema" (str "/schema/index.html")) ;TODO version (toplink "Library" "/library")] [:div#accordian.accordian From 31dce4b9c860981c6fffc993f393ab105ae2b27d Mon Sep 17 00:00:00 2001 From: Mike Travers Date: Sat, 4 Mar 2023 17:10:50 -0800 Subject: [PATCH 10/19] embed working again --- README-devops.org | 5 +++++ doc/tutorial.org | 14 +++++++------- src/clj/org/parkerici/enflame/embed_server.clj | 2 +- src/cljs/org/parkerici/enflame/core.cljs | 11 ++++++++++- src/cljs/org/parkerici/enflame/embed.cljs | 6 +++--- 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/README-devops.org b/README-devops.org index 7c705ed..611f5a5 100644 --- a/README-devops.org +++ b/README-devops.org @@ -10,6 +10,11 @@ Once App Runner is set up, pushing a new Docker image will restart the server. bin/build.sh does all the necessary steps +** Preconditions +- Docker must be running +- AWS cred must be in place +- emacs available from shell + * AWS Services diff --git a/doc/tutorial.org b/doc/tutorial.org index 0e35eb5..d1e6359 100644 --- a/doc/tutorial.org +++ b/doc/tutorial.org @@ -17,7 +17,7 @@ When building a query, you select blocks that represent the type of object you are interested in, and other blocks that represent attribiutes of those objects, or relationships to other objects. To start with a simple example, here' a query that will find all the =subject= objects with race =african-american=: #+BEGIN_EXPORT html - + #+END_EXPORT Click the [[file:Screen_Shot_2021-05-14_at_12.04.21_PM.png]] button to run the query, and observe the results. @@ -33,7 +33,7 @@ Next, try changing the query. Here are some things you can try: Here's a slightly more complex query. Again we are searching for subjects, but now we have multiple constraints on the attributes. The query starts off with two constraints...try running it. #+BEGIN_EXPORT html - + #+END_EXPORT Note that there is a third, unconnected attribute block [[file:Screen_Shot_2021-05-15_at_11.57.09_AM.png]] lying around. It doesn't have any effect until you add it to the query – try doing that by dragging in under the ... block and running the query again. @@ -51,7 +51,7 @@ TODO example has position problems # [subjects where [sample is [samples]] and [race is white]] #+BEGIN_EXPORT html - + #+END_EXPORT Now try constraining the subquery by dragging the [[file:Screen_Shot_2021-05-12_at_4.55.29_PM.png]] block into the sample query block and running the query again. @@ -64,7 +64,7 @@ TODO this should be output everything probably, # [variants where gene is EGFR] #+BEGIN_EXPORT html - + #+END_EXPORT You can try substituting a different gene name. @@ -76,7 +76,7 @@ This example shows how you can produce row-specific counts. This query finds all # Subjects with disease (counts #+BEGIN_EXPORT html - + #+END_EXPORT Try adding the [[file:Screen_Shot_2021-05-17_at_9.41.13_AM.png]] block to constrain diseases based on a substring of the disease name. @@ -88,7 +88,7 @@ Here's a more complex example. This query returns subjects that have a variant i # subjects where variant is variants where gene is EGFR #+BEGIN_EXPORT html - + #+END_EXPORT Try changing the gene, or experimenting with the outputs. @@ -99,7 +99,7 @@ Try changing the gene, or experimenting with the outputs. Enflame can not only generate queries, but generate data visualizations of the results. Here's an example which summarizes the counts of clinical observations. #+BEGIN_EXPORT html - + #+END_EXPORT diff --git a/src/clj/org/parkerici/enflame/embed_server.clj b/src/clj/org/parkerici/enflame/embed_server.clj index e3ad4ba..2f470c5 100644 --- a/src/clj/org/parkerici/enflame/embed_server.clj +++ b/src/clj/org/parkerici/enflame/embed_server.clj @@ -31,7 +31,7 @@ [:div.scrollbar.scrollbar-primary {:style {:height "300px" :width "100%" :overflow-y "scroll"}} [:div#results.force-overflow]]]] [:script {:src "js/compiled/app.js"}]] - [:script "org.parkerici.enflame.embed.init();"] + [:script "org.parkerici.enflame.core.embed();"] ])) "text/html")) diff --git a/src/cljs/org/parkerici/enflame/core.cljs b/src/cljs/org/parkerici/enflame/core.cljs index 07ff315..c0b9b41 100644 --- a/src/cljs/org/parkerici/enflame/core.cljs +++ b/src/cljs/org/parkerici/enflame/core.cljs @@ -8,7 +8,7 @@ [org.parkerici.enflame.views :as views] [org.parkerici.enflame.db] [org.parkerici.enflame.config :as config] - org.parkerici.enflame.embed + [org.parkerici.enflame.embed :as embed] org.parkerici.enflame.schema-client )) @@ -95,3 +95,12 @@ (ag/init) ) +;;; This used to be in embed but that has dependency issues +(defn ^:export embed + [] + (config/init #(do + (custom-init (config/config)) + (embed/re-frame-init) + (embed/init-embed) + )) + (ag/init)) diff --git a/src/cljs/org/parkerici/enflame/embed.cljs b/src/cljs/org/parkerici/enflame/embed.cljs index f76b004..81b5a37 100644 --- a/src/cljs/org/parkerici/enflame/embed.cljs +++ b/src/cljs/org/parkerici/enflame/embed.cljs @@ -8,6 +8,7 @@ [org.parkerici.enflame.views :as views] [org.parkerici.enflame.view.graph :as graph] [org.parkerici.enflame.db] + [org.parkerici.enflame.config :as config] )) ;;; Alternative to core for embedded @@ -47,15 +48,14 @@ "data" views/results}) (views/results))]) -(defn ^:export re-frame-init +(defn re-frame-init [] (rf/dispatch-sync [:initialize-db]) (rf/clear-subscription-cache!)) ;; Probably this would be better if blockly was a real react component. -(defn ^:export init +(defn init-embed [] - (re-frame-init) (let [{:keys [ddb query view rows] :as _params} (browser/url-params)] (let [blockly-id "blocklyEmbed" results-id "results" From f575fa3be0245eb119c180f381205c9744792cd8 Mon Sep 17 00:00:00 2001 From: Mike Travers Date: Mon, 6 Mar 2023 16:24:28 -0800 Subject: [PATCH 11/19] More deployment tweaks and documentation --- README-devops.org | 31 ------ README.md | 4 +- bin/build.sh | 9 +- deploy/README-devops.org | 59 +++++++++++ deploy/candel-config.edn | 18 ++++ deploy/resources | 117 +++++++++++++++++++++ project.clj | 4 +- src/cljs/org/parkerici/enflame/config.cljs | 2 +- src/cljs/org/parkerici/enflame/views.cljs | 7 +- 9 files changed, 209 insertions(+), 42 deletions(-) delete mode 100644 README-devops.org create mode 100644 deploy/README-devops.org create mode 100644 deploy/candel-config.edn create mode 100644 deploy/resources diff --git a/README-devops.org b/README-devops.org deleted file mode 100644 index 611f5a5..0000000 --- a/README-devops.org +++ /dev/null @@ -1,31 +0,0 @@ -* Scope of this document - - - -* Configuration - -* Deployment - -Once App Runner is set up, pushing a new Docker image will restart the server. - -bin/build.sh does all the necessary steps - -** Preconditions -- Docker must be running -- AWS cred must be in place -- emacs available from shell - - -* AWS Services - -These were all cobbled together by hand, I suppose the TODO is to make a CloudFormation them. - -** Datomic Cloud - -** App Runner - -** ECR (Container Registry) - -** IAM - -** TODO storage service for library diff --git a/README.md b/README.md index 71e2090..51ea3b0 100644 --- a/README.md +++ b/README.md @@ -24,11 +24,11 @@ See [sample config](resources/candel-config.edn) ## To run locally from source: -Copy `resources/candel-config.edn` to `deploy/candel-config.edn`, filling out as appropriate. +Create the file `deploy/launch-config.edn` according to your needs (see the sample configs in `resources`). Then: lein launch -This will compile the front-end, lauch a server, and open a browser windoe. +This will compile the front-end, lauch the back-end, and open a browser windoe. ## Documentation generation diff --git a/bin/build.sh b/bin/build.sh index 70b5542..ce2efa9 100755 --- a/bin/build.sh +++ b/bin/build.sh @@ -1,5 +1,8 @@ # Build Docker image +# ID of AWS Elastic Container Registry +export ECR=733151965047.dkr.ecr.us-east-1.amazonaws.com + # TODO parameterize properly export VERSION=$(git rev-parse --short HEAD) @@ -27,10 +30,10 @@ docker build -t cbio . # docker run -p 8080 cbio -docker tag cbio:latest 733151965047.dkr.ecr.us-east-1.amazonaws.com/cbio:latest +docker tag cbio:latest $ECR/cbio:latest # Upload to AWS repository -aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 733151965047.dkr.ecr.us-east-1.amazonaws.com +aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $ECR -docker push 733151965047.dkr.ecr.us-east-1.amazonaws.com/cbio:latest +docker push $ECR/cbio:latest diff --git a/deploy/README-devops.org b/deploy/README-devops.org new file mode 100644 index 0000000..b5e3b95 --- /dev/null +++ b/deploy/README-devops.org @@ -0,0 +1,59 @@ +* Scope of this document + +This describes the simplified CANDEL/Enflame architecture used for the public CandelBIO website. + +* Deployment + +Once App Runner is set up, pushing a new Docker image will reboot the server. + +[[bin/build.sh]] does all the necessary steps + +** Preconditions +- Docker must be running +- AWS cred must be in place +- emacs available from shell + + +* AWS Services + +These were all cobbled together by hand, I suppose the TODO is to make a CloudFormation them. See [[deploy/resources]] for more details. + +** Datomic Cloud + +Launched from AWS Console: https://docs.datomic.com/cloud/getting-started/start-system.html + +This resuls in creating three CloudFormation stacks, one master and two subordinates. + +Current master is https://us-east-1.console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/stackinfo?filteringText=&filteringStatus=active&viewNested=true&stackId=arn%3Aaws%3Acloudformation%3Aus-east-1%3A733151965047%3Astack%2FPublicCANDEL5%2F0eee3a40-b863-11ed-9e49-1270ef971d67 + +You need to copy the ClientApiGatewayEnpoint output to Enflame's configuration. The current value is https://nazpex6ueb.execute-api.us-east-1.amazonaws.com + +** ECR (Container Registry) + + +This holds the Docker container that App Runner uses. + +Current instance: https://us-east-1.console.aws.amazon.com/ecr/repositories/private/733151965047/cbio?region=us-east-1 + +The name of the most recent image is: 733151965047.dkr.ecr.us-east-1.amazonaws.com/cbio:latest +This needs to be supplied to the build.sh script + +** IAM + +Service role for App Runner + +https://us-east-1.console.aws.amazon.com/iamv2/home?region=us-east-1#/roles/details/Enflame-OpenCANDEL?section=permissions + +** App Runner + +App Runner service has to be configured with the + + +https://us-east-1.console.aws.amazon.com/apprunner/home?region=us-east-1#/services/dashboard?service_arn=arn%3Aaws%3Aapprunner%3Aus-east-1%3A733151965047%3Aservice%2FEnflame-OpenCANDEL%2F68f5614bbe7248d2896d397b1e3b2033&active_tab=logs + +Service: Enflame-OpenCANDEL +Public endpint: https://7gbxx4vxvn.us-east-1.awsapprunner.com/ + + + +** TODO storage service for library diff --git a/deploy/candel-config.edn b/deploy/candel-config.edn new file mode 100644 index 0000000..e6221af --- /dev/null +++ b/deploy/candel-config.edn @@ -0,0 +1,18 @@ +;;; TODO Use aero to default to resources/candel-config.edn +{:source {:type :candel + :candelabra-endpoint "https://pici-prod-v1.candel.parkerici.org/" + :insecure-https false} + :schema "resources/candel-schema-1-3-1.edn" + :query-generator :candel-generate + :library {:gcs-project "pici-dev" + :gcs-console-url "https://console.cloud.google.com/datastore/entities;kind=EnflameItem;ns=__$DEFAULT$__;sortCol=date-created;sortDir=DESCENDING/query/kind?project=pici-dev"} + :rh-cards [:candel/db + :query + :candel/wick + :share + :compact ;debug only + :browser + ] + :dev? true ;TODO control this somehow maybe aero/env + :port 1983 + } diff --git a/deploy/resources b/deploy/resources new file mode 100644 index 0000000..c9da67c --- /dev/null +++ b/deploy/resources @@ -0,0 +1,117 @@ +Note: this is a dump of relevant AWS resources (results trimmed to only include the OpenCANDEL relevant ones) + + +14:49:59 ~/os/enflame (opencandel) ⪢ aws apprunner list-services +{ + "ServiceSummaryList": [ + { + "ServiceName": "Enflame-OpenCANDEL", + "ServiceId": "68f5614bbe7248d2896d397b1e3b2033", + "ServiceArn": "arn:aws:apprunner:us-east-1:733151965047:service/Enflame-OpenCANDEL/68f5614bbe7248d2896d397b1e3b2033", + "ServiceUrl": "7gbxx4vxvn.us-east-1.awsapprunner.com", + "CreatedAt": "2023-03-01T16:01:27-08:00", + "UpdatedAt": "2023-03-01T16:01:27-08:00", + "Status": "RUNNING" + } + ] +} + +15:03:07 ~/os/enflame (opencandel) ⪢ aws iam list-roles +{ + "Roles": [ + { + "Path": "/", + "RoleName": "Enflame-OpenCANDEL", + "RoleId": "AROA2VM2DF53ZHMLN5XGO", + "Arn": "arn:aws:iam::733151965047:role/Enflame-OpenCANDEL", + "CreateDate": "2023-03-02T00:58:08+00:00", + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + }, + "Action": "sts:AssumeRole" + }, + { + "Effect": "Allow", + "Principal": { + "Service": "tasks.apprunner.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] + }, + "Description": "Role for Enflame OpenCANDEL server", + "MaxSessionDuration": 3600 + } + ] +} + + + +14:52:19 ~/os/enflame (opencandel) ⪢ aws ecr describe-repositories +{ + "repositories": [ + + { + "repositoryArn": "arn:aws:ecr:us-east-1:733151965047:repository/cbio", + "registryId": "733151965047", + "repositoryName": "cbio", + "repositoryUri": "733151965047.dkr.ecr.us-east-1.amazonaws.com/cbio", + "createdAt": "2023-03-01T13:32:19-08:00", + "imageTagMutability": "MUTABLE", + "imageScanningConfiguration": { + "scanOnPush": false + }, + "encryptionConfiguration": { + "encryptionType": "AES256" + } + } + ] +} + + +14:50:11 ~/os/enflame (opencandel) ⪢ aws cloudformation list-stacks +{ + "StackSummaries": [ + { + "StackId": "arn:aws:cloudformation:us-east-1:733151965047:stack/PublicCANDEL5-Compute-93S9RT2L5TUI/bbcf46f0-b863-11ed-932d-0ab2700ca4cf", + "StackName": "PublicCANDEL5-Compute-93S9RT2L5TUI", + "TemplateDescription": "Creates compute resources needed to run Datomic.", + "CreationTime": "2023-03-01T19:03:21.404000+00:00", + "StackStatus": "CREATE_COMPLETE", + "ParentId": "arn:aws:cloudformation:us-east-1:733151965047:stack/PublicCANDEL5/0eee3a40-b863-11ed-9e49-1270ef971d67", + "RootId": "arn:aws:cloudformation:us-east-1:733151965047:stack/PublicCANDEL5/0eee3a40-b863-11ed-9e49-1270ef971d67", + "DriftInformation": { + "StackDriftStatus": "NOT_CHECKED" + } + }, + { + "StackId": "arn:aws:cloudformation:us-east-1:733151965047:stack/PublicCANDEL5-StorageF7F305E7-UD47ELG33U9Z/134d9cc0-b863-11ed-b08e-128bb3d0467b", + "StackName": "PublicCANDEL5-StorageF7F305E7-UD47ELG33U9Z", + "TemplateDescription": "Creates storage resources needed to run Datomic.", + "CreationTime": "2023-03-01T18:58:38.643000+00:00", + "StackStatus": "CREATE_COMPLETE", + "ParentId": "arn:aws:cloudformation:us-east-1:733151965047:stack/PublicCANDEL5/0eee3a40-b863-11ed-9e49-1270ef971d67", + "RootId": "arn:aws:cloudformation:us-east-1:733151965047:stack/PublicCANDEL5/0eee3a40-b863-11ed-9e49-1270ef971d67", + "DriftInformation": { + "StackDriftStatus": "NOT_CHECKED" + } + }, + { + "StackId": "arn:aws:cloudformation:us-east-1:733151965047:stack/PublicCANDEL5/0eee3a40-b863-11ed-9e49-1270ef971d67", + "StackName": "PublicCANDEL5", + "TemplateDescription": "Creates resources needed to run Datomic. The stack name is the name of the Datomic system.", + "CreationTime": "2023-03-01T18:58:31.189000+00:00", + "StackStatus": "CREATE_COMPLETE", + "DriftInformation": { + "StackDriftStatus": "NOT_CHECKED" + } + }, + + ] +} + diff --git a/project.clj b/project.clj index 6e66b19..3b3c55d 100644 --- a/project.clj +++ b/project.clj @@ -75,12 +75,10 @@ :min-lein-version "2.5.3" :main ^:skip-aot org.parkerici.enflame.core - ;; This builds and launches the CANDEL version - ;;; note: you have to fill out the config file in resources/candel-config.edn and put it in deploy folder :aliases {"launch" ["do" "clean" ["cljsbuild" "once" "prod"] - ["run" "deploy/candel-config.edn"]]} + ["run" "deploy/launch-config.edn"]]} :source-paths ["src/clj" "src/cljs" "src/cljc"] :test-paths ["test/clj" "test/cljs" "test/cljc"] diff --git a/src/cljs/org/parkerici/enflame/config.cljs b/src/cljs/org/parkerici/enflame/config.cljs index 0439db5..5c535ce 100644 --- a/src/cljs/org/parkerici/enflame/config.cljs +++ b/src/cljs/org/parkerici/enflame/config.cljs @@ -22,7 +22,7 @@ (defonce the-config (atom nil)) (defn config - ([key] (get @the-config key)) + ([& keys] (get-in @the-config keys)) ([] @the-config )) ;;; Get the config from the server diff --git a/src/cljs/org/parkerici/enflame/views.cljs b/src/cljs/org/parkerici/enflame/views.cljs index fd93c72..3c16de3 100644 --- a/src/cljs/org/parkerici/enflame/views.cljs +++ b/src/cljs/org/parkerici/enflame/views.cljs @@ -333,11 +333,14 @@ [:div {:style {:margin-top "10px"}} [:div ;; TODO customize these - (toplink [:span [:b "Enflame "] [:img {:src "favicon.ico" :width "24px"}]] "http://github.com/ParkerICI/enflame") + (toplink [:span [:b "Enflame "] [:img {:src "favicon.ico" :width "24px"}]] "http://github.com/CandelBio/enflame") (toplink "Tutorial" "/doc/tutorial.html") (toplink "Doc" "/doc/guide.html") (toplink "Schema" (str "/schema/index.html")) ;TODO version - (toplink "Library" "/library")] + + (when (c/config :library :type) + (toplink "Library" "/library")) + ] [:div#accordian.accordian (for [card (c/config :rh-cards)] ;; cards TODO not working yet because timing From d8cc3bbee0cfd13979c0dde8cda66420ffb67396 Mon Sep 17 00:00:00 2001 From: Mike Travers Date: Thu, 9 Mar 2023 12:22:12 -0800 Subject: [PATCH 12/19] DynamoDB for library storage -- mostly working --- deploy/README-devops.org | 6 ++ project.clj | 7 ++ resources/opencandel-config.edn | 7 +- .../org/parkerici/enflame/library/core.clj | 21 ++++- .../parkerici/enflame/library/dynamodb.clj | 88 +++++++++++++++++++ .../org/parkerici/enflame/library/view.clj | 3 +- 6 files changed, 124 insertions(+), 8 deletions(-) create mode 100644 src/clj/org/parkerici/enflame/library/dynamodb.clj diff --git a/deploy/README-devops.org b/deploy/README-devops.org index b5e3b95..16de048 100644 --- a/deploy/README-devops.org +++ b/deploy/README-devops.org @@ -54,6 +54,12 @@ https://us-east-1.console.aws.amazon.com/apprunner/home?region=us-east-1#/servic Service: Enflame-OpenCANDEL Public endpint: https://7gbxx4vxvn.us-east-1.awsapprunner.com/ +** DynamoDB + +For library storage + +https://us-east-1.console.aws.amazon.com/dynamodbv2/home?region=us-east-1#table?name=OpenCANDEL_EnflameLibrary + ** TODO storage service for library diff --git a/project.clj b/project.clj index 3b3c55d..4a35539 100644 --- a/project.clj +++ b/project.clj @@ -26,6 +26,8 @@ [ring-middleware-format "0.7.5" :exclusions [javax.xml.bind/jaxb-api]] [bk/ring-gzip "0.3.0"] [trptcolin/versioneer "0.2.0"] + + ;; Note: the gcs stuff is not going to used post PICI and could be removed. So annoying that it is hard to have different configs. [com.google.cloud/google-cloud-datastore "1.105.7" ;; TODO have Cognitect or someone review this :exclusions [com.google.errorprone/error_prone_annotations @@ -33,6 +35,11 @@ org.apache.httpcomponents/httpclient com.google.guava/guava com.fasterxml.jackson.core/jackson-core]] + + [com.cognitect.aws/api "0.8.652"] + [com.cognitect.aws/endpoints "1.1.12.415"] + [com.cognitect.aws/dynamodb "825.2.1262.0"] + [environ "1.1.0"] [me.raynes/fs "1.4.6"] [org.parkerici/multitool "0.0.26"] diff --git a/resources/opencandel-config.edn b/resources/opencandel-config.edn index 3964fdd..31498ad 100644 --- a/resources/opencandel-config.edn +++ b/resources/opencandel-config.edn @@ -8,7 +8,9 @@ :schema "candel-schema-1-3-1.edn" ;filename of an Alzabo schema - :library {} ;TODO need an AWS library solution + :library {:type :dynamodb + + } :query-generator :candel-generate ;The query generator @@ -16,8 +18,7 @@ :query ;; I guess no point to this until Wick server is brought up ;; :candel/wick - ;; Library also temporarily out of service - ;; :share + :share ;; :compact ;debug only :browser ] diff --git a/src/clj/org/parkerici/enflame/library/core.clj b/src/clj/org/parkerici/enflame/library/core.clj index 00ad443..0e81761 100644 --- a/src/clj/org/parkerici/enflame/library/core.clj +++ b/src/clj/org/parkerici/enflame/library/core.clj @@ -1,5 +1,6 @@ (ns org.parkerici.enflame.library.core (:require [org.parkerici.enflame.gcs :as gcs] + [org.parkerici.enflame.library.dynamodb :as dyna] [org.parkerici.enflame.config :as config] ) ) @@ -10,10 +11,22 @@ (defn upload [item] - (case (config/config [:library :type]) - :gcs (gcs/upload "EnflameItem" item big-keys))) + (case (config/config :library :type) + :gcs (gcs/upload "EnflameItem" item big-keys) + :dynamodb (dyna/upload item) + )) (defn get-item [key] - (case (config/config [:library :type]) - :gcs (gcs/get-item "EnflameItem" (Long. key)))) + (case (config/config :library :type) + :gcs (gcs/get-item "EnflameItem" (Long. key)) + :dynamodb (dyna/get-item key) + )) + +(defn list-items + [] + (case (config/config :library :type) + :gcs (gcs/list-items "EnflameItem") + :dynamodb (dyna/list-items) + ) + ) diff --git a/src/clj/org/parkerici/enflame/library/dynamodb.clj b/src/clj/org/parkerici/enflame/library/dynamodb.clj new file mode 100644 index 0000000..d460968 --- /dev/null +++ b/src/clj/org/parkerici/enflame/library/dynamodb.clj @@ -0,0 +1,88 @@ +(ns org.parkerici.enflame.library.dynamodb + (:require [cognitect.aws.client.api :as aws] + [org.parkerici.enflame.library.item :as item] + [org.parkerici.enflame.config :as config] + [org.parkerici.multitool.core :as u] + ) + ) + +(def dyna (aws/client {:api :dynamodb})) + +(def table "OpenCANDEL_EnflameLibrary") ;TODO config + +;;; Delete the table +(defn delete-table + [] + (aws/invoke dyna + {:op :DeleteTable + :request {:TableName table}})) + + +#_ + (mapv (fn [k] + {:AttributeName (name k) + :AttributeType "S"}) + (item/item-keys)) + +(defn create-table + [] + (u/ignore-report + (delete-table)) + (aws/invoke dyna + {:op :CreateTable + :request {:TableName table + ;; Note: only key fields should be defined here + :AttributeDefinitions [{:AttributeName "entityId" + :AttributeType "S"} + ] + + :KeySchema [{:AttributeName "entityId" + :KeyType "HASH"} + #_ {:AttributeName "foo" + :KeyType "RANGE"}] + :ProvisionedThroughput {:ReadCapacityUnits 1 + :WriteCapacityUnits 1}}})) + + +(defn from-item + [item] + (-> (into {} + (map (fn [[k v]] + [(keyword "org.parkerici.enflame.library.item" (name k)) + (or (:N v) (:S v))]) + item)) + (update :org.parkerici.enflame.library.item/date-created + #(-> % parse-long)))) ;view calls java.util.Date. + +(defn list-items + [] + (map from-item + (:Items + (aws/invoke dyna {:op :Scan :request {:TableName table}})))) + +(defn to-items + [thing] + (let [thang (assoc thing :entityId (str (java.util.UUID/randomUUID))) + ks (keys thang)] + (zipmap ks + (map #(let [v (% thang)] + (cond (number? v) + {:N (str v)} + :else + {:S (str v)})) + ks)))) + +(defn upload + [thing] + (aws/invoke + dyna + {:op :PutItem + :request {:TableName table + :Item (to-items thing)}}) ) + +(defn get-item + [k] + (-> (aws/invoke dyna {:op :GetItem :request {:TableName table :Key {"entityId" {:S k}}}}) + :Item + from-item)) + diff --git a/src/clj/org/parkerici/enflame/library/view.clj b/src/clj/org/parkerici/enflame/library/view.clj index 9fa25fa..8d0b81a 100644 --- a/src/clj/org/parkerici/enflame/library/view.clj +++ b/src/clj/org/parkerici/enflame/library/view.clj @@ -1,6 +1,7 @@ (ns org.parkerici.enflame.library.view (:require [org.parkerici.enflame.html :as html] [org.parkerici.enflame.gcs :as gcs] + [org.parkerici.enflame.library.core :as lib] [org.parkerici.enflame.library.item :as item] [org.parkerici.enflame.config :as config] [org.parkerici.multitool.core :as u] @@ -28,7 +29,7 @@ (defn library-items [] - (map item/localize-item (gcs/list-items "EnflameItem"))) + (map item/localize-item (lib/list-items))) (defn view [] From 0976a79e703bf198fc51c4751c91eff2c940042e Mon Sep 17 00:00:00 2001 From: Mike Travers Date: Thu, 9 Mar 2023 12:55:58 -0800 Subject: [PATCH 13/19] Get table from config, cleanup --- resources/opencandel-config.edn | 2 +- .../parkerici/enflame/library/dynamodb.clj | 30 +++++++------------ 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/resources/opencandel-config.edn b/resources/opencandel-config.edn index 31498ad..1c08871 100644 --- a/resources/opencandel-config.edn +++ b/resources/opencandel-config.edn @@ -9,7 +9,7 @@ :schema "candel-schema-1-3-1.edn" ;filename of an Alzabo schema :library {:type :dynamodb - + :table "OpenCANDEL_EnflameLibrary" } :query-generator :candel-generate ;The query generator diff --git a/src/clj/org/parkerici/enflame/library/dynamodb.clj b/src/clj/org/parkerici/enflame/library/dynamodb.clj index d460968..ebf4632 100644 --- a/src/clj/org/parkerici/enflame/library/dynamodb.clj +++ b/src/clj/org/parkerici/enflame/library/dynamodb.clj @@ -8,29 +8,22 @@ (def dyna (aws/client {:api :dynamodb})) -(def table "OpenCANDEL_EnflameLibrary") ;TODO config +(defn table + [] + (config/config :library :table)) -;;; Delete the table +;;; Delete the table – beware, data loss! (defn delete-table [] (aws/invoke dyna - {:op :DeleteTable - :request {:TableName table}})) - - -#_ - (mapv (fn [k] - {:AttributeName (name k) - :AttributeType "S"}) - (item/item-keys)) + {:op :DeleteTable + :request {:TableName (table)}})) (defn create-table [] - (u/ignore-report - (delete-table)) (aws/invoke dyna {:op :CreateTable - :request {:TableName table + :request {:TableName (table) ;; Note: only key fields should be defined here :AttributeDefinitions [{:AttributeName "entityId" :AttributeType "S"} @@ -38,8 +31,7 @@ :KeySchema [{:AttributeName "entityId" :KeyType "HASH"} - #_ {:AttributeName "foo" - :KeyType "RANGE"}] + ] :ProvisionedThroughput {:ReadCapacityUnits 1 :WriteCapacityUnits 1}}})) @@ -58,7 +50,7 @@ [] (map from-item (:Items - (aws/invoke dyna {:op :Scan :request {:TableName table}})))) + (aws/invoke dyna {:op :Scan :request {:TableName (table)}})))) (defn to-items [thing] @@ -77,12 +69,12 @@ (aws/invoke dyna {:op :PutItem - :request {:TableName table + :request {:TableName (table) :Item (to-items thing)}}) ) (defn get-item [k] - (-> (aws/invoke dyna {:op :GetItem :request {:TableName table :Key {"entityId" {:S k}}}}) + (-> (aws/invoke dyna {:op :GetItem :request {:TableName (table) :Key {"entityId" {:S k}}}}) :Item from-item)) From 2f5133fd376a643bf1ef7c303e2b3ac76ae52637 Mon Sep 17 00:00:00 2001 From: Mike Travers Date: Tue, 14 Mar 2023 20:27:29 -0700 Subject: [PATCH 14/19] library refinements --- deploy/resources | 65 ++++++++++++++++++- resources/opencandel-config.edn | 1 + .../parkerici/enflame/library/dynamodb.clj | 48 ++++++++------ .../org/parkerici/enflame/library/view.clj | 4 +- .../org/parkerici/enflame/candel/query.cljc | 3 + src/cljc/org/parkerici/enflame/schema.cljc | 4 ++ 6 files changed, 101 insertions(+), 24 deletions(-) diff --git a/deploy/resources b/deploy/resources index c9da67c..03f4612 100644 --- a/deploy/resources +++ b/deploy/resources @@ -1,4 +1,4 @@ -Note: this is a dump of relevant AWS resources (results trimmed to only include the OpenCANDEL relevant ones) +Note: this is a dump of relevant AWS resources (results trimmed to only include the OpenCANDEL relevax`nt ones) 14:49:59 ~/os/enflame (opencandel) ⪢ aws apprunner list-services @@ -52,6 +52,40 @@ Note: this is a dump of relevant AWS resources (results trimmed to only include +18:23:09 ~/os/enflame (opencandel) ⪢ aws iam list-role-policies --role-name "Enflame-OpenCANDEL" +{ + "PolicyNames": [ + "OpenCANDEL_Enflame_Library" + ] +} + + + +18:22:49 ~/os/enflame (opencandel) ⪢ aws iam get-role-policy --role-name "Enflame-OpenCANDEL" --policy-name "OpenCANDEL_Enflame_Library" +{ + "RoleName": "Enflame-OpenCANDEL", + "PolicyName": "OpenCANDEL_Enflame_Library", + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "dynamodb:PutItem", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:Query", + "dynamodb:UpdateItem" + ], + "Resource": "arn:aws:dynamodb:us-east-1:733151965047:table/OpenCANDEL_EnflameLibrary" + } + ] + } +} + + + 14:52:19 ~/os/enflame (opencandel) ⪢ aws ecr describe-repositories { "repositories": [ @@ -115,3 +149,32 @@ Note: this is a dump of relevant AWS resources (results trimmed to only include ] } +20:25:31 ~/os/enflame (opencandel) ⪢ aws dynamodb describe-table --table-name "OpenCANDEL_EnflameLibrary" +{ + "Table": { + "AttributeDefinitions": [ + { + "AttributeName": "entityId", + "AttributeType": "S" + } + ], + "TableName": "OpenCANDEL_EnflameLibrary", + "KeySchema": [ + { + "AttributeName": "entityId", + "KeyType": "HASH" + } + ], + "TableStatus": "ACTIVE", + "CreationDateTime": "2023-03-09T09:57:35.493000-08:00", + "ProvisionedThroughput": { + "NumberOfDecreasesToday": 0, + "ReadCapacityUnits": 1, + "WriteCapacityUnits": 1 + }, + "TableSizeBytes": 57488, + "ItemCount": 3, + "TableArn": "arn:aws:dynamodb:us-east-1:733151965047:table/OpenCANDEL_EnflameLibrary", + "TableId": "f29723c1-b287-48da-8c8b-6739caaed63e" + } +} diff --git a/resources/opencandel-config.edn b/resources/opencandel-config.edn index 1c08871..823f86d 100644 --- a/resources/opencandel-config.edn +++ b/resources/opencandel-config.edn @@ -10,6 +10,7 @@ :library {:type :dynamodb :table "OpenCANDEL_EnflameLibrary" + :console-url "https://us-east-1.console.aws.amazon.com/dynamodbv2/home?region=us-east-1#table?name=OpenCANDEL_EnflameLibrary" } :query-generator :candel-generate ;The query generator diff --git a/src/clj/org/parkerici/enflame/library/dynamodb.clj b/src/clj/org/parkerici/enflame/library/dynamodb.clj index ebf4632..c0866da 100644 --- a/src/clj/org/parkerici/enflame/library/dynamodb.clj +++ b/src/clj/org/parkerici/enflame/library/dynamodb.clj @@ -8,6 +8,13 @@ (def dyna (aws/client {:api :dynamodb})) +(defn invoke-with-eh + [& args] + (let [resp (apply aws/invoke dyna args)] + (when (:cognitect.anomalies/category resp) + (throw (ex-info "DynamoDB exception" {:args args :resp resp}))) + resp)) + (defn table [] (config/config :library :table)) @@ -15,25 +22,25 @@ ;;; Delete the table – beware, data loss! (defn delete-table [] - (aws/invoke dyna - {:op :DeleteTable - :request {:TableName (table)}})) + (invoke-with-eh + {:op :DeleteTable + :request {:TableName (table)}})) (defn create-table [] - (aws/invoke dyna - {:op :CreateTable - :request {:TableName (table) - ;; Note: only key fields should be defined here - :AttributeDefinitions [{:AttributeName "entityId" - :AttributeType "S"} - ] + (invoke-with-eh + {:op :CreateTable + :request {:TableName (table) + ;; Note: only key fields should be defined here + :AttributeDefinitions [{:AttributeName "entityId" + :AttributeType "S"} + ] - :KeySchema [{:AttributeName "entityId" - :KeyType "HASH"} - ] - :ProvisionedThroughput {:ReadCapacityUnits 1 - :WriteCapacityUnits 1}}})) + :KeySchema [{:AttributeName "entityId" + :KeyType "HASH"} + ] + :ProvisionedThroughput {:ReadCapacityUnits 1 + :WriteCapacityUnits 1}}})) (defn from-item @@ -50,9 +57,9 @@ [] (map from-item (:Items - (aws/invoke dyna {:op :Scan :request {:TableName (table)}})))) + (invoke-with-eh {:op :Scan :request {:TableName (table)}})))) -(defn to-items +(defn to-item [thing] (let [thang (assoc thing :entityId (str (java.util.UUID/randomUUID))) ks (keys thang)] @@ -66,15 +73,14 @@ (defn upload [thing] - (aws/invoke - dyna + (invoke-with-eh {:op :PutItem :request {:TableName (table) - :Item (to-items thing)}}) ) + :Item (to-item thing)}}) ) (defn get-item [k] - (-> (aws/invoke dyna {:op :GetItem :request {:TableName (table) :Key {"entityId" {:S k}}}}) + (-> (invoke-with-eh dyna {:op :GetItem :request {:TableName (table) :Key {"entityId" {:S k}}}}) :Item from-item)) diff --git a/src/clj/org/parkerici/enflame/library/view.clj b/src/clj/org/parkerici/enflame/library/view.clj index 8d0b81a..04c7035 100644 --- a/src/clj/org/parkerici/enflame/library/view.clj +++ b/src/clj/org/parkerici/enflame/library/view.clj @@ -36,8 +36,8 @@ (html/html-frame "Library" [:div.container.col-12 - [:a {:href (config/config :library :gcs-console-url)} - "Edit in GCS console"] + [:a {:href (config/config :library :console-url)} + "Edit in cloud console"] [:table.table [:tbody (for [item (reverse (sort-by ::item/date-created (library-items)))] diff --git a/src/cljc/org/parkerici/enflame/candel/query.cljc b/src/cljc/org/parkerici/enflame/candel/query.cljc index 980ac72..11b3efb 100644 --- a/src/cljc/org/parkerici/enflame/candel/query.cljc +++ b/src/cljc/org/parkerici/enflame/candel/query.cljc @@ -4,6 +4,9 @@ [org.parkerici.multitool.core :as u] [clojure.string :as str])) +;;; For dev +;;; (schema/set-schema (org.parkerici.enflame.config/read-schema nil)) + ;;; ⊓⊔⊓⊔ Query logic ⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔ (def varcounter (atom {})) diff --git a/src/cljc/org/parkerici/enflame/schema.cljc b/src/cljc/org/parkerici/enflame/schema.cljc index fd8e1b5..6bef1c7 100644 --- a/src/cljc/org/parkerici/enflame/schema.cljc +++ b/src/cljc/org/parkerici/enflame/schema.cljc @@ -36,6 +36,10 @@ [k] (get-in @the-schema [:kinds k :fields])) +(defn kind-field + [k f] + (get-in @the-schema [:kinds k :fields f])) + (defn kind? [k] (contains? (:kinds @the-schema) k)) From 853af8105f3cf8cb1953e5078a9b20ed095abfe7 Mon Sep 17 00:00:00 2001 From: Mike Travers Date: Fri, 17 Mar 2023 13:25:17 -0700 Subject: [PATCH 15/19] tests working again, fix bug in inverse relations --- src/clj/org/parkerici/enflame/config.clj | 27 +++++---- src/clj/org/parkerici/enflame/server.clj | 4 +- src/cljc/org/parkerici/enflame/blockdefs.cljc | 56 ++++++++++++------- .../org/parkerici/enflame/blockdefs_test.clj | 2 +- .../org/parkerici/enflame/library_test.clj | 5 +- test/clj/org/parkerici/enflame/query_test.clj | 2 +- test/clj/org/parkerici/enflame/test_utils.clj | 15 ++--- 7 files changed, 67 insertions(+), 44 deletions(-) diff --git a/src/clj/org/parkerici/enflame/config.clj b/src/clj/org/parkerici/enflame/config.clj index 2607601..f736e2a 100644 --- a/src/clj/org/parkerici/enflame/config.clj +++ b/src/clj/org/parkerici/enflame/config.clj @@ -3,6 +3,7 @@ [clojure.edn :as edn] [clojure.pprint :as pprint] [org.parkerici.multitool.core :as u] + [org.parkerici.enflame.schema :as schema] [clojure.java.io :as io] )) @@ -10,25 +11,29 @@ (def the-config (atom nil)) +(defn config + [& keys] + (assert @the-config "Config not set") + (get-in @the-config keys)) + +;;; Called from server +;;; Here because schema is cljc +(defn read-schema [from] + (-> from + io/resource + slurp + edn/read-string)) + (defn load-config [file] (let [env-config {} ; if we want to supply some config from environment file-config (edn/read-string (slurp (io/resource file))) config (merge env-config file-config)] (pprint/pprint config) - (reset! the-config config))) + (reset! the-config config) + (schema/set-schema (read-schema (config :schema))))) -(defn config - ([& keys] (get-in @the-config keys)) - ([] @the-config )) -;;; Called from server -;;; TODO use version -(u/defn-memoized read-schema [version] - (-> (config :schema) - io/resource - slurp - edn/read-string)) ;;; TODO diff --git a/src/clj/org/parkerici/enflame/server.clj b/src/clj/org/parkerici/enflame/server.clj index 3edb1c9..0c6417a 100644 --- a/src/clj/org/parkerici/enflame/server.clj +++ b/src/clj/org/parkerici/enflame/server.clj @@ -133,8 +133,8 @@ (GET "/config" req (response/response (config/config))) (GET "/databases" req ;TODO candel specific. Fold into schema (response/response (handle-databases req config))) - (GET "/schema" [version] - (response/response (config/read-schema version))) + (GET "/schema" [version] ;TODO version ignored + (response/response (config/read-schema))) (GET "/query" req (handle-query req config)) (context "/library" [] (GET "/get" [key] diff --git a/src/cljc/org/parkerici/enflame/blockdefs.cljc b/src/cljc/org/parkerici/enflame/blockdefs.cljc index f06203a..139f4b9 100644 --- a/src/cljc/org/parkerici/enflame/blockdefs.cljc +++ b/src/cljc/org/parkerici/enflame/blockdefs.cljc @@ -150,7 +150,7 @@ {:message0 (str (name field) "? %1") :args0 [{:name "V" :type "field_dropdown" - :options [["yes" :true] ["no" :false] ["defined" :any]]}] + :options '[["yes" true] ["no" false] ["defined" any]]}] :query-builder :query-primitive-field} (:long :float) {:message0 (str (name field) " %2 %1") @@ -220,23 +220,42 @@ (defn field-def [kind field] (get-in (schema/kind-def kind) [:fields field])) -(defn kind-field-blockdef - [kind field invert?] - (let [{:keys [type attribute]} (if invert? ;??? not sure about this - {:type field :attribute (keyword field kind)} - (field-def kind field))] + +(defn base-block + [kind field attribute] + {:type (str (name kind) "_" (name field)) + :previousStatement (str (name kind) "_constraint") + :nextStatement (str (name kind) "_constraint") + :colour (kind-color kind) + :helpUrl (alzabo-url kind) + :attribute (or attribute (keyword (name kind) (name field))) + :input (or (:type (field-def kind field)) + field) + }) + + +;;; eg :measurement :measurement-set {:type :measurement-set, :cardinality :many, :doc "The measurement values for this measurement set", :attribute :measurement-set/measurements} +(defn kind-field-inverse-blockdef + [kind {:keys [type attribute]}] + (let [field type] + (merge + {:invert? true} + (field-def-type field field) + (base-block kind field attribute)))) + + +(defn kind-field-forward-blockdef + [kind field] + ;; TODO this is wrong, some attributes are plural eg :measurement-set/measurements + (let [{:keys [type attribute]} (field-def kind field)] (when-let [field-def (field-def-type field type)] (merge field-def - {:type (str (name kind) "_" (name field)) - :previousStatement (str (name kind) "_constraint") - :nextStatement (str (name kind) "_constraint") - :colour (kind-color kind) - :helpUrl (alzabo-url kind) - :attribute (or attribute (keyword (name kind) (name field))) - :input type - :invert? invert? - })))) + (base-block kind field attribute) + )))) + + + ;;; ⊓⊔⊓⊔ Complex relations ⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔ @@ -354,12 +373,11 @@ (concat ;; forward relations (map (fn [field] - (kind-field-blockdef kind field false)) + (kind-field-forward-blockdef kind field)) (keys (schema/kind-fields kind))) ;; inverse relations - (map (fn [[other-kind adef]] - (let [adef (first adef)] - (kind-field-blockdef kind other-kind true))) + (map (fn [[other-kind [adef]]] + (kind-field-inverse-blockdef kind adef)) (kind (schema/inverse-relations))) (kind-complex-blockdefs kind)) (filter identity) diff --git a/test/clj/org/parkerici/enflame/blockdefs_test.clj b/test/clj/org/parkerici/enflame/blockdefs_test.clj index f192f8e..c53ae33 100644 --- a/test/clj/org/parkerici/enflame/blockdefs_test.clj +++ b/test/clj/org/parkerici/enflame/blockdefs_test.clj @@ -5,7 +5,7 @@ [org.parkerici.enflame.schema :as schema] )) -(use-fixtures :once with-schema) +(use-fixtures :once with-test-config) (deftest blockdef-test (testing "blockdefs for single kind (subject)" diff --git a/test/clj/org/parkerici/enflame/library_test.clj b/test/clj/org/parkerici/enflame/library_test.clj index fef8f7a..e9dbf25 100644 --- a/test/clj/org/parkerici/enflame/library_test.clj +++ b/test/clj/org/parkerici/enflame/library_test.clj @@ -7,7 +7,7 @@ [clojure.test :as t])) -(use-fixtures :once with-schema) +(use-fixtures :once with-test-config) ;;; Turn off annoying print-map feature (defmethod print-method clojure.lang.IPersistentMap [m, ^java.io.Writer w] @@ -53,6 +53,7 @@ :dquery {:find ((pull ?subject1 [:db/id :subject/id])), :where ([?subject1 :subject/id ?id1])} } + {:query "[samples where [tumor-type is *]]", :compact {:type "sample_query", @@ -88,8 +89,6 @@ :where ([?subject1 :subject/meddra-disease ?meddra-disease1])}} - - {:query "[samples where [gdc-anatomic-site is [gdc-anatomic-sites]]]", :compact {:type "sample_query", diff --git a/test/clj/org/parkerici/enflame/query_test.clj b/test/clj/org/parkerici/enflame/query_test.clj index 8197438..1f9bf07 100644 --- a/test/clj/org/parkerici/enflame/query_test.clj +++ b/test/clj/org/parkerici/enflame/query_test.clj @@ -3,7 +3,7 @@ [org.parkerici.enflame.test-utils :refer :all] [org.parkerici.enflame.candel.query :refer :all])) -(use-fixtures :once with-schema) +(use-fixtures :once with-test-config) (defn set= [a b] (and (= (set a) (set b)) diff --git a/test/clj/org/parkerici/enflame/test_utils.clj b/test/clj/org/parkerici/enflame/test_utils.clj index c942d19..730180c 100644 --- a/test/clj/org/parkerici/enflame/test_utils.clj +++ b/test/clj/org/parkerici/enflame/test_utils.clj @@ -1,14 +1,15 @@ (ns org.parkerici.enflame.test-utils (:require [clojure.test :refer :all] - [org.parkerici.enflame.server :as server] - [org.parkerici.enflame.schema :as schema] [org.parkerici.enflame.config :as config] )) -(use-fixtures :once - (config/load-config "test/resources/test-config.edn")) +(defn with-test-config + [f] + (config/load-config "test/test-config.edn") + (f)) + +;;; Put this in individual test files +#_ (use-fixtures :once with-test-config) + -(defn with-schema [f] - (schema/set-schema (config/read-schema nil nil)) ;TODO args - (f)) From a375061720a49d72dc07c3cac6e1d1592c7cc3ea Mon Sep 17 00:00:00 2001 From: Mike Travers Date: Fri, 17 Mar 2023 16:18:19 -0700 Subject: [PATCH 16/19] fix bug in inverse relations, other tweaks --- .../org/parkerici/enflame/library/dynamodb.clj | 15 +++++++++++---- src/clj/org/parkerici/enflame/server.clj | 3 +-- src/cljc/org/parkerici/enflame/blockdefs.cljc | 2 +- src/cljc/org/parkerici/enflame/candel/query.cljc | 3 --- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/clj/org/parkerici/enflame/library/dynamodb.clj b/src/clj/org/parkerici/enflame/library/dynamodb.clj index c0866da..3c4ff2a 100644 --- a/src/clj/org/parkerici/enflame/library/dynamodb.clj +++ b/src/clj/org/parkerici/enflame/library/dynamodb.clj @@ -8,13 +8,20 @@ (def dyna (aws/client {:api :dynamodb})) +;;; Invoke with error handling (defn invoke-with-eh - [& args] - (let [resp (apply aws/invoke dyna args)] + [arg-map] + (let [resp (aws/invoke dyna arg-map)] (when (:cognitect.anomalies/category resp) - (throw (ex-info "DynamoDB exception" {:args args :resp resp}))) + (throw (ex-info "DynamoDB exception" {:arg-map arg-map :resp resp}))) resp)) +;;; When debugging, might want to Turn off the magic +#_ +(defn invoke-with-eh + [arg-map] + (aws/invoke dyna arg-map)) + (defn table [] (config/config :library :table)) @@ -80,7 +87,7 @@ (defn get-item [k] - (-> (invoke-with-eh dyna {:op :GetItem :request {:TableName (table) :Key {"entityId" {:S k}}}}) + (-> (invoke-with-eh {:op :GetItem :request {:TableName (table) :Key {"entityId" {:S k}}}}) :Item from-item)) diff --git a/src/clj/org/parkerici/enflame/server.clj b/src/clj/org/parkerici/enflame/server.clj index 0c6417a..16f943f 100644 --- a/src/clj/org/parkerici/enflame/server.clj +++ b/src/clj/org/parkerici/enflame/server.clj @@ -43,7 +43,6 @@ _query (read-string query) _args (if (u/nullish? args) [] (read-string args)) _limit (if (u/nullish? limit) nil (Integer. limit)) - candelabra-token (get-in req [:cookies "candelabra-token" :value]) results #_ (datomic/query db _query _args candelabra-token config) (datomic-client/query db _query _args) clipped (if _limit (take _limit results) results)] @@ -134,7 +133,7 @@ (GET "/databases" req ;TODO candel specific. Fold into schema (response/response (handle-databases req config))) (GET "/schema" [version] ;TODO version ignored - (response/response (config/read-schema))) + (response/response @schema/the-schema)) (GET "/query" req (handle-query req config)) (context "/library" [] (GET "/get" [key] diff --git a/src/cljc/org/parkerici/enflame/blockdefs.cljc b/src/cljc/org/parkerici/enflame/blockdefs.cljc index 139f4b9..0f3c75a 100644 --- a/src/cljc/org/parkerici/enflame/blockdefs.cljc +++ b/src/cljc/org/parkerici/enflame/blockdefs.cljc @@ -150,7 +150,7 @@ {:message0 (str (name field) "? %1") :args0 [{:name "V" :type "field_dropdown" - :options '[["yes" true] ["no" false] ["defined" any]]}] + :options '[["yes" "true"] ["no" "false"] ["defined" "any"]]}] :query-builder :query-primitive-field} (:long :float) {:message0 (str (name field) " %2 %1") diff --git a/src/cljc/org/parkerici/enflame/candel/query.cljc b/src/cljc/org/parkerici/enflame/candel/query.cljc index 11b3efb..980ac72 100644 --- a/src/cljc/org/parkerici/enflame/candel/query.cljc +++ b/src/cljc/org/parkerici/enflame/candel/query.cljc @@ -4,9 +4,6 @@ [org.parkerici.multitool.core :as u] [clojure.string :as str])) -;;; For dev -;;; (schema/set-schema (org.parkerici.enflame.config/read-schema nil)) - ;;; ⊓⊔⊓⊔ Query logic ⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔⊓⊔ (def varcounter (atom {})) From c53568d6766f6e32d1440e32a7cb0f87f007dc1f Mon Sep 17 00:00:00 2001 From: Mike Travers Date: Thu, 23 Mar 2023 09:38:19 -0700 Subject: [PATCH 17/19] kludge to fix weird longs being returned from transit --- project.clj | 10 ++++++---- src/cljs/org/parkerici/enflame/datomic.cljs | 14 +++++++++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/project.clj b/project.clj index 4a35539..95527c4 100644 --- a/project.clj +++ b/project.clj @@ -36,7 +36,7 @@ com.google.guava/guava com.fasterxml.jackson.core/jackson-core]] - [com.cognitect.aws/api "0.8.652"] + [com.cognitect.aws/api "0.8.656"] [com.cognitect.aws/endpoints "1.1.12.415"] [com.cognitect.aws/dynamodb "825.2.1262.0"] @@ -48,7 +48,9 @@ [org.clojure/clojurescript "1.10.520"] [aristotle/aristotle "0.1.0" - :exclusions [org.apache.jena/apache-jena-libs]] ;asking for trouble + :exclusions [org.apache.jena/apache-jena-libs ;asking for trouble + javax.xml.bind/jaxb-api + ]] [org.apache.jena/apache-jena-libs "3.16.0" :extension "pom"] [metasoarous/oz "1.6.0-alpha6" ; warning: later versions seem to have broken dependencies :exclusions [cljsjs/vega ; we insert a later version of Vega to fix some bugs @@ -62,7 +64,7 @@ [cljsjs/vega-embed "6.19.0-0"] [cljsjs/vega-tooltip "0.27.0-0"] - [org.parkerici/blockoid "0.3.6"] + [org.parkerici/blockoid "0.3.10"] [reagent "0.8.1"] [re-frame "0.10.9" :exclusions [org.clojure/clojurescript]] @@ -70,7 +72,7 @@ [cljsjs/ag-grid-enterprise "25.3.0-1"] [inflections "0.13.2"] [com.andrewmcveigh/cljs-time "0.5.2"] - [cljs-ajax "0.8.0"]] + [cljs-ajax "0.8.4"]] :repositories [["github" {:url "https://maven.pkg.github.com/ParkerICI/mvn-packages" :sign-releases false :username :env/github_user diff --git a/src/cljs/org/parkerici/enflame/datomic.cljs b/src/cljs/org/parkerici/enflame/datomic.cljs index 789114e..621edfe 100644 --- a/src/cljs/org/parkerici/enflame/datomic.cljs +++ b/src/cljs/org/parkerici/enflame/datomic.cljs @@ -3,9 +3,20 @@ [org.parkerici.multitool.core :as u] [org.parkerici.enflame.api :as api] [org.parkerici.enflame.results :as results] + [clojure.walk :as walk] [cljs.reader :as reader] )) +;;; Note: for some reason, transit is producing goog.long objectts instead of proper clojure longs. Can't figure out why, this patches the problem +(defn fix-results + [r] + (walk/postwalk + (fn [x] + (if (.-low_ x) + (long x) + x)) + r)) + ;;; TODO this isn't all that Datomic specific, maybe rename @@ -21,7 +32,8 @@ (api/ajax-get "/api/query" (merge {:url-params (u/clean-map {:db ddb :query (str query) :limit limit :args (str args)}) - :handler handler} + :handler (fn [r] + (handler (fix-results r)))} (or options {})))) From 4eb383d2a2c6f4d04fae96f255f4156aa7d7e29e Mon Sep 17 00:00:00 2001 From: Mike Travers Date: Thu, 23 Mar 2023 13:38:26 -0700 Subject: [PATCH 18/19] quick hack query optimizer --- .../org/parkerici/enflame/datomic_client.clj | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/clj/org/parkerici/enflame/datomic_client.clj b/src/clj/org/parkerici/enflame/datomic_client.clj index c808125..ac08fbb 100644 --- a/src/clj/org/parkerici/enflame/datomic_client.clj +++ b/src/clj/org/parkerici/enflame/datomic_client.clj @@ -14,11 +14,33 @@ [db] (d/connect @client {:db-name db})) -(defn query - [db query args] - (d/q {:query query :args (cons (d/db (conn db)) args)})) + (defn dbs [] (d/list-databases @client {})) +;;; Query optimizer +;;; Cute hack, and no idea why Datomic doesn't just do this + +(u/defn-memoized db-stats + [db] + (d/db-stats (d/db (conn db)))) + +(defn db-att-count + [db att] + (get-in (db-stats db) [:attrs att :count] 0)) + +;;; Resorts where clauses so smaller searches come first +(defn optimize-query + [db query] + (update query :where + (fn [clauses] + (sort-by (comp (partial db-att-count db) second) + clauses)))) + + +(defn query + [db query args] + (d/q {:query (optimize-query db query) + :args (cons (d/db (conn db)) args)})) From f9c87bb5f4f83428acb83b3b7832568079d56581 Mon Sep 17 00:00:00 2001 From: Mike Travers Date: Tue, 28 Mar 2023 10:44:49 -0700 Subject: [PATCH 19/19] Undid some of last change which was breaking something fundamental --- bin/build.sh | 2 +- project.clj | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/bin/build.sh b/bin/build.sh index ce2efa9..f988d34 100755 --- a/bin/build.sh +++ b/bin/build.sh @@ -18,7 +18,7 @@ bin/build-guide.sh # Run Alzabo to build schemas # cd alzabo; lein with-profile prod do clean, run documentation candel "*", cljsbuild once; cd .. -lein uberjar +lein do clean, uberjar # TODO abort if lein fails, duh. diff --git a/project.clj b/project.clj index 95527c4..51627a9 100644 --- a/project.clj +++ b/project.clj @@ -49,7 +49,8 @@ [aristotle/aristotle "0.1.0" :exclusions [org.apache.jena/apache-jena-libs ;asking for trouble - javax.xml.bind/jaxb-api + ;; TODO this fixed a problem with longs, but caused other issues... + ; javax.xml.bind/jaxb-api ]] [org.apache.jena/apache-jena-libs "3.16.0" :extension "pom"] [metasoarous/oz "1.6.0-alpha6" ; warning: later versions seem to have broken dependencies @@ -72,7 +73,7 @@ [cljsjs/ag-grid-enterprise "25.3.0-1"] [inflections "0.13.2"] [com.andrewmcveigh/cljs-time "0.5.2"] - [cljs-ajax "0.8.4"]] + [cljs-ajax "0.8.0"]] :repositories [["github" {:url "https://maven.pkg.github.com/ParkerICI/mvn-packages" :sign-releases false :username :env/github_user @@ -142,7 +143,8 @@ :compiler {:main org.parkerici.enflame.core :output-to "resources/public/js/compiled/app.js" :output-dir "resources/public/js/compiled/outprod" - :optimizations :simple - :infer-externs true + :asset-path "js/compiled/outprod" + :optimizations :none ; was :simple but that stopped working for unknown reasons + :infer-externs true :closure-defines {goog.DEBUG false} :pretty-print false}}]})