diff --git a/artifacts/yprovstore.yml b/artifacts/yprovstore.yml new file mode 100644 index 00000000..664f9642 --- /dev/null +++ b/artifacts/yprovstore.yml @@ -0,0 +1,147 @@ +--- +- name: Install yProvStore + hosts: localhost + connection: local + vars: + branch: "v2.0.1" + traefik_letsencrypt_email: "{{ letsencrypt_email | default('server@yprovstore.fedcloud.eu') }}" + traefik_dns_name: "{{ dns_name | default('yprovstore.fedcloud.eu') }}" + traefik_public_ip: "{{ public_ip | default(ansible_default_ipv4.address) }}" + minio_endpoint: "localhost:9000" + minio_root_password: "minio123" + minio_root_user: "minio" + minio_bucket_name: "yprov-documents" + minio_secure: "False" + roles: + - role: 'grycap.docker' + tasks: + - name: check if yProvStore files are already downloaded + stat: + path: /opt/yProvStore/docker-compose.yml + register: docker_file_stat + + - name: Download yProvStore files + git: + repo: 'https://github.com/HPCI-Lab/yProvStore' + dest: /opt/yProvStore + version: "{{ branch }}" + when: not docker_file_stat.stat.exists + + - name: copy .env.example to .env + copy: + src: /opt/yProvStore/.env.example + dest: /opt/yProvStore/.env + remote_src: true + mode: '644' + force: false + + - name: Create keys directory + file: + path: /opt/yProvStore/keys/ + state: directory + mode: '644' + + - name: Create private key file for + copy: + content: "" + dest: /opt/yProvStore/keys/user_private.pem + mode: '644' + force: false + + - name: Set MinIO connection variables + set_fact: + minio_endpoint: "{{ storage_connection_string | urlsplit('hostname') }}" + minio_root_password: "{{ storage_connection_string | urlsplit('password') }}" + minio_root_user: "{{ storage_connection_string | urlsplit('username') }}" + minio_bucket_name: "{{ storage_connection_string | urlsplit('path') | regex_replace('^/', '') }}" + minio_secure: "True" + when: storage_connection_string is defined and storage_connection_string != '' + + - name: Update .env file + lineinfile: + path: /opt/yProvStore/.env + regexp: '^{{ item.var }}=' + line: '{{ item.var }}={{ item.value }}' + loop: + - { var: 'USE_LOCAL_PID_SERVICE', value: 'True' } + - { var: 'MINIO_ROOT_USER', value: '{{ minio_root_user }}' } + - { var: 'MINIO_ROOT_PASSWORD', value: '{{ minio_root_password }}' } + - { var: 'MINIO_BUCKET_NAME', value: '{{ minio_bucket_name }}' } + - { var: 'MINIO_ENDPOINT', value: '{{ minio_endpoint }}' } + - { var: 'MINIO_SECURE', value: '{{ minio_secure }}' } + - { var: 'POSTGRES_USER', value: 'postgres' } + - { var: 'POSTGRES_PASSWORD', value: 'postgres123' } + - { var: 'POSTGRES_DB', value: 'yprovstore' } + - { var: 'APP_PORT', value: '8000' } + - { var: 'DOCUMENT_SIZE_LIMIT', value: '{{ document_size_limit | default(100) }}' } + + - name: Update .env file + lineinfile: + path: /opt/yProvStore/.env + regexp: '^DB_CONNECTION_STRING=' + line: 'DB_CONNECTION_STRING={{ db_connection_string }}' + when: db_connection_string is defined and db_connection_string != '' + + - name: Update .env file with OIDC authentication variables + lineinfile: + path: /opt/yProvStore/.env + regexp: '^{{ item.var }}=' + line: '{{ item.var }}={{ item.value }}' + loop: + - { var: 'USE_EGI_CHECKIN_AUTH', value: 'True' } + - { var: 'EGI_CHECKIN_INTROSPECTION_ENDPOINT', value: '{{ oidc_instrospection_endpoint }}' } + - { var: 'EGI_CHECKIN_CLIENT_ID', value: '{{ oidc_client_id }}' } + - { var: 'EGI_CHECKIN_CLIENT_SECRET', value: '{{ oidc_client_secret }}' } + - { var: 'EGI_CHECKIN_REQUIRED_ENTITLEMENTS', value: '{{ oidc_required_entitlements }}' } + when: use_oidc_auth | bool | default(false) + + - name: Update docker-compose.yml with Traefik configuration + blockinfile: + path: /opt/yProvStore/docker-compose.yml + insertbefore: '^ minio:' + prepend_newline: true + block: | + # Traefik reverse proxy configuration + labels: + - "traefik.enable=true" + - "traefik.http.routers.api.service=api" + - "traefik.http.routers.api.rule=Host(`{{ traefik_dns_name }}`, `{{ traefik_public_ip }}`)" + - "traefik.http.routers.api.entrypoints=websecure" + - "traefik.http.routers.api.tls.certresolver=myresolver" + - "traefik.http.services.api.loadbalancer.server.port=8000" + traefik: + image: "traefik:v2.11" + container_name: "traefik" + command: + # - "--log.level=DEBUG" + - "--api.insecure=true" + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.web.address=:80" + - "--entrypoints.websecure.address=:443" + - "--entrypoints.web.http.redirections.entryPoint.to=websecure" + - "--entrypoints.web.http.redirections.entryPoint.scheme=https" + - "--entrypoints.web.http.redirections.entrypoint.permanent=true" + - "--certificatesresolvers.myresolver.acme.httpchallenge=true" + - "--certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web" + #- "--certificatesresolvers.myresolver.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory" + - "--certificatesresolvers.myresolver.acme.email={{ traefik_letsencrypt_email }}" + - "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json" + ports: + - "80:80" + - "443:443" + - "8080:8080" + volumes: + - "./letsencrypt:/letsencrypt" + - "/var/run/docker.sock:/var/run/docker.sock:ro" + + - name: Add curl package in Dockerfile + lineinfile: + path: /opt/yProvStore/Dockerfile + line: ' curl \' + insertafter: ' dos2unix \\' + + - name: Exec docker-compose up + docker_compose: + project_src: /opt/yProvStore/ + state: present diff --git a/templates/simple-node-disk.yml b/templates/simple-node-disk.yml index 347837df..7cc11d2e 100644 --- a/templates/simple-node-disk.yml +++ b/templates/simple-node-disk.yml @@ -60,6 +60,7 @@ metadata: - interoperability-registry-read.yml - pyhandle.yml - apptainer.yaml + - yprovstore.yaml topology_template: diff --git a/templates/yprovstore.yaml b/templates/yprovstore.yaml new file mode 100644 index 00000000..2e2daaae --- /dev/null +++ b/templates/yprovstore.yaml @@ -0,0 +1,139 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +imports: + - grycap_custom_types: https://raw.githubusercontent.com/grycap/tosca/main/custom_types.yaml + + +description: Deploy yProvStore, a provenance store for scientific workflows. + +metadata: + template_version: "1.0.0" + template_name: yProvStore + display_name: Deploy yProvStore + web: https://github.com/HPCI-Lab/yProvStore + parents: + - simple-node-disk.yml + tabs: + yProvStore: + - letsencrypt_email + - db_connection_string + - storage_connection_string + - document_size_limit + - use_oidc_auth + - oidc_instrospection_endpoint: + enabled_by: use_oidc_auth + - oidc_required_entitlements: + enabled_by: use_oidc_auth + - oidc_client_id: + enabled_by: use_oidc_auth + - oidc_client_secret: + enabled_by: use_oidc_auth + +topology_template: + inputs: + + letsencrypt_email: + default: changeme@email.com + description: Email to be used for Let's Encrypt certificates + type: string + + db_connection_string: + default: '' + description: | + Database connection string for yProvStore (format postgresql://user:pass@pgserver/db). + If not provided, a local PostgreSQL instance will be deployed and used. + type: string + storage_connection_string: + default: '' + description: | + Storage connection string for yProvStore (format s3://access_key:secret_key@endpoint/bucket). + If not provided, a local MinIO instance will be deployed and used. + type: string + document_size_limit: + default: 100 + description: Maximum size in MB of documents that can be stored in yProvStore + type: integer + + # Restrict some simple-node-disk input values + + storage_size: + type: scalar-unit.size + description: Size of the extra HD added to the instance (Set 0 if disk is not needed) + default: 50 GiB + constraints: + - valid_values: [ 50 GiB, 100 GiB, 200 GiB, 500 GiB, 1 TiB, 2 TiB, 10 TiB, 20 TiB, 40 TiB, 100 TiB ] + + mount_path: + type: string + description: Path to mount the extra disk + default: /data + + use_oidc_auth: + type: boolean + description: Whether to use OIDC authentication for yProvStore + default: false + oidc_instrospection_endpoint: + type: string + description: OIDC introspection endpoint + default: 'https://enes-proxy.pilot.eosc-beyond.eu/auth/realms/enes/protocol/openid-connect/token/introspect' + oidc_required_entitlements: + type: string + description: Required entitlement to access yProvStore + default: 'urn:mace:egi.eu:group:enes.pilot.eosc-beyond.eu:role=member#aai.egi.eu' + oidc_client_id: + type: string + description: OIDC Client ID + default: '' + oidc_client_secret: + type: string + description: OIDC Client Secret + default: '' + + dns_name: + default: yprovstore-test.vm.fedcloud.eu + description: DNS hostname to access yProvStore + type: string + + node_templates: + + yprovstore: + type: tosca.nodes.ec3.Application + artifacts: + docker_role: + file: grycap.docker + type: tosca.artifacts.AnsibleGalaxy.role + capabilities: + endpoint: + properties: + ports: + port_http: + protocol: tcp + source: 80 + port_https: + protocol: tcp + source: 443 + interfaces: + Standard: + configure: + implementation: https://raw.githubusercontent.com/grycap/tosca/eosc_beyond/artifacts/yprovstore.yml + inputs: + letsencrypt_email: { get_input: letsencrypt_email } + dns_name: { get_input: dns_name } + public_ip: { get_attribute: [ simple_node, public_address, 0 ] } + db_connection_string: { get_input: db_connection_string } + storage_connection_string: { get_input: storage_connection_string } + document_size_limit: { get_input: document_size_limit } + use_oidc_auth: { get_input: use_oidc_auth } + oidc_instrospection_endpoint: { get_input: oidc_instrospection_endpoint } + oidc_required_entitlements: { get_input: oidc_required_entitlements } + oidc_client_id: { get_input: oidc_client_id } + oidc_client_secret: { get_input: oidc_client_secret } + + requirements: + - host: simple_node + + outputs: + yprovstore_url: + value: { concat: [ 'https://', get_attribute: [ simple_node, public_address, 0 ], '/docs' ] } + yprovstore_url_dns: + value: { concat: [ 'https://', get_input: [ dns_name ], '/docs' ] }