Updates required for a stable deployment

docs/configuration.md

@@ -201,7 +201,7 @@ respectively.
 
     - `copy_files_all`
     - `copy_files_[ansible_group_name]`
-    - `copy_files_[ansible_host_name]`
+    - `copy_files_[ansible_ssh_host_name]`
 
 Copying two files/folders onto the hpc02-test node could be done by
 setting the following ansible variable e.g. in the

docs/installation.md

@@ -61,7 +61,7 @@ Below is an example ansible inventory file used for
 `ansible-playbook`. There are [great docs on modifying the ansible
 inventory
 file](https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html). The important keys to set:
-  - `ansible_host` which is the DNS accessible name
+  - `ansible_ssh_host` which is the DNS accessible name
   - `ansible_port` default is `22`
   - `ansible_user` which is the username to login into node by default is the user that the `ansible-playbook` command is run as
   - `ansible_ssh_private_key_file` is the path to the ssh key to login to node
@@ -72,10 +72,10 @@ group. `N` nodes can be in the `hpc_worker` section (including the
 hpc_master node which is not recommended).
 
 ```
-hpc02-test ansible_host=192.168.121.124 ansible_port=22 ansible_user='vagrant' ansible_ssh_private_key_file='/home/costrouc/.vagrant.d/insecure_private_key'
-hpc03-test ansible_host=192.168.121.176 ansible_port=22 ansible_user='vagrant' ansible_ssh_private_key_file='/home/costrouc/.vagrant.d/insecure_private_key'
-hpc04-test ansible_host=192.168.121.133 ansible_port=22 ansible_user='vagrant' ansible_ssh_private_key_file='/home/costrouc/.vagrant.d/insecure_private_key'
-hpc01-test ansible_host=192.168.121.35 ansible_port=22 ansible_user='vagrant' ansible_ssh_private_key_file='/home/costrouc/.vagrant.d/insecure_private_key'
+hpc02-test ansible_ssh_host=192.168.121.124 ansible_port=22 ansible_user='vagrant' ansible_ssh_private_key_file='/home/costrouc/.vagrant.d/insecure_private_key'
+hpc03-test ansible_ssh_host=192.168.121.176 ansible_port=22 ansible_user='vagrant' ansible_ssh_private_key_file='/home/costrouc/.vagrant.d/insecure_private_key'
+hpc04-test ansible_ssh_host=192.168.121.133 ansible_port=22 ansible_user='vagrant' ansible_ssh_private_key_file='/home/costrouc/.vagrant.d/insecure_private_key'
+hpc01-test ansible_ssh_host=192.168.121.35 ansible_port=22 ansible_user='vagrant' ansible_ssh_private_key_file='/home/costrouc/.vagrant.d/insecure_private_key'
 
 [hpc_master]
 hpc01-test

playbook.yaml

@@ -48,8 +48,8 @@
 
      - name: Keycloak administration credentials
        ansible.builtin.debug:
-         msg: "Keycloak administration username={{ keycloak_admin_username }} via https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}/auth/admin/"
+         msg: "Keycloak administration username={{ keycloak_admin_username }} via https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}/auth/admin/"
 
      - name: Accessing cluster
        ansible.builtin.debug:
-         msg: "Access cluster via following url: https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}"
+         msg: "Access cluster via following url: https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}"

roles/conda_store/tasks/conda_store.yaml

@@ -56,7 +56,7 @@
          protocol: openid-connect
          protocolMapper: oidc-group-membership-mapper
      redirect_uris:
-       - "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}/conda-store/oauth_callback"
+       - "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}/conda-store/oauth_callback"
    register: conda_store_client
 
  - name: Create conda-store keycloak roles

roles/conda_store/templates/conda_store_config.py

@@ -19,7 +19,7 @@
 
 c.S3Storage.internal_endpoint = "localhost:{{ minio_internal_port }}"
 c.S3Storage.internal_secure = False
-c.S3Storage.external_endpoint = "{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}:{{ minio_external_port }}"
+c.S3Storage.external_endpoint = "{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}:{{ minio_external_port }}"
 c.S3Storage.external_secure = True
 c.S3Storage.access_key = "{{ minio_username }}"
 c.S3Storage.secret_key = "{{ minio_password }}"
@@ -42,10 +42,10 @@
 # ==================================
 #         auth settings
 # ==================================
-c.GenericOAuthAuthentication.access_token_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}/auth/realms/{{ keycloak_realm }}/protocol/openid-connect/token"
-c.GenericOAuthAuthentication.authorize_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}/auth/realms/{{ keycloak_realm }}/protocol/openid-connect/auth"
-c.GenericOAuthAuthentication.user_data_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}/auth/realms/{{ keycloak_realm }}/protocol/openid-connect/userinfo"
-c.GenericOAuthAuthentication.oauth_callback_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}/conda-store/oauth_callback"
+c.GenericOAuthAuthentication.access_token_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}/auth/realms/{{ keycloak_realm }}/protocol/openid-connect/token"
+c.GenericOAuthAuthentication.authorize_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}/auth/realms/{{ keycloak_realm }}/protocol/openid-connect/auth"
+c.GenericOAuthAuthentication.user_data_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}/auth/realms/{{ keycloak_realm }}/protocol/openid-connect/userinfo"
+c.GenericOAuthAuthentication.oauth_callback_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}/conda-store/oauth_callback"
 c.GenericOAuthAuthentication.client_id = "{{ conda_store_client_id }}"
 c.GenericOAuthAuthentication.client_secret = "{{ conda_store_client_secret }}"
 c.GenericOAuthAuthentication.access_scope = "profile"

roles/conda_store/templates/environments/conda-store.yaml

@@ -3,4 +3,5 @@ channels:
  - conda-forge
 dependencies:
  - conda-store-server=={{ conda_store_version }}
- - pymysql
+ - pymysql==1.0.2
+ - pydantic<2.0

roles/dask_gateway/tasks/client.yaml

@@ -12,7 +12,7 @@
      content: |
        gateway:
          address: "http://{{ groups['hpc_master'][0] }}:{{ dask_gateway_api_port }}"
-         public-address:  "https://{{ hostvars[groups['hpc_master'][0]].ansible_host }}"
+         public-address:  "https://{{ hostvars[groups['hpc_master'][0]].ansible_ssh_host }}"
          proxy-address: "tls://{{ groups['hpc_master'][0] }}:{{ dask_gateway_scheduler_external_port }}"
          auth:
            type: jupyterhub

roles/grafana/tasks/grafana.yaml

@@ -49,7 +49,7 @@
          protocol: openid-connect
          protocolMapper: oidc-group-membership-mapper
      redirect_uris:
-       - "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}{{ grafana_base_url }}/login/generic_oauth"
+       - "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}{{ grafana_base_url }}/login/generic_oauth"
    register: grafana_client
 
  - name: Create grafana keycloak roles
@@ -131,7 +131,7 @@
        [server]
        protocol = http
        http_port = {{ grafana_port }}
-       domain = {{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}
+       domain = {{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}
        root_url = https://%(domain)s{{ grafana_base_url }}
        serve_from_sub_path = true
 
@@ -148,9 +148,9 @@
        client_id = {{ grafana_client_id }}
        client_secret = {{ grafana_client_secret }}
        scopes = profile
-       auth_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}/auth/realms/{{ keycloak_realm }}/protocol/openid-connect/auth"
-       token_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}/auth/realms/{{ keycloak_realm }}/protocol/openid-connect/token"
-       api_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}/auth/realms/{{ keycloak_realm }}/protocol/openid-connect/userinfo"
+       auth_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}/auth/realms/{{ keycloak_realm }}/protocol/openid-connect/auth"
+       token_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}/auth/realms/{{ keycloak_realm }}/protocol/openid-connect/token"
+       api_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}/auth/realms/{{ keycloak_realm }}/protocol/openid-connect/userinfo"
        tls_skip_verify_insecure = true
        login_attribute_path: preferred_username
        role_attribute_path: "contains(roles[*], 'grafana_admin') && 'Admin' || contains(roles[*], 'grafana_developer') && 'Editor' || contains(roles[*], 'grafana_viewer') || 'Viewer'"

roles/jupyterhub/tasks/jupyterhub.yaml

@@ -69,7 +69,7 @@
          protocol: openid-connect
          protocolMapper: oidc-group-membership-mapper
      redirect_uris:
-       - "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}/hub/oauth_callback"
+       - "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}/hub/oauth_callback"
    register: jupyterhub_client
 
  - name: Create JupyterHub Keycloak roles

roles/jupyterhub/templates/environments/dashboards.yaml

@@ -5,9 +5,9 @@ dependencies:
   - pip
   - nb_conda_kernels
   # required to launch jupterlab from jupyterhub
-  - jupyterhub
+  - jupyterhub==2.3.0
   # jupyterhub menu https://github.com/jupyterlab/jupyterlab/issues/9428
-  - jupyterlab>=3.0.4
+  - jupyterlab>=3.4.0
   - ipywidgets>=7.6.0
   - ipyparallel
   # dashboards (versions specified to narrow conda solve space)
@@ -16,6 +16,6 @@ dependencies:
   - voila >= 0.2.7
   - streamlit >= 0.76
   - dash >= 1.19
-  - cdsdashboards-singleuser
+  - cdsdashboards-singleuser==0.6.3
   - pip:
       - batchspawner==1.1.0

roles/jupyterhub/templates/environments/jupyterhub.yaml

@@ -2,17 +2,18 @@ name: jupyterhub
 channels:
   - conda-forge
 dependencies:
-  - pip
-  - jupyterhub{{ jupyterhub_version }}
-  - oauthenticator
-  - jupyterhub-idle-culler
-  - jupyterhub-systemdspawner
-  - python-keycloak
+  - pip==21.1.2
+  - jupyterhub==2.3.0
+  - jupyterhub-kubespawner==1.1.0
+  - oauthenticator==14.1.0
+  - escapism==1.0.1
+  - cdsdashboards==0.6.3
+  - jupyterhub-idle-culler==1.0
   - pip:
-      - qhub-jupyterhub-theme==0.3.1
-      - jupyterhub-traefik-proxy
+      - nebari_jupyterhub_theme==2023.4.1
+      - sqlalchemy==1.4.4
+      - python-keycloak==0.26.1
+      - jupyterhub-traefik-proxy==0.3.0
+      - batchspawner==1.1.0
       # jupyterhub-ssh has not made a release yet
       - git+https://github.com/yuvipanda/jupyterhub-ssh.git
-      - batchspawner==1.1.0
-      # cdsdashboards
-      - cdsdashboards==0.5.0b2

roles/jupyterhub/templates/environments/jupyterlab.yaml

@@ -2,22 +2,51 @@ name: jupyterlab
 channels:
   - conda-forge
 dependencies:
+# general
   - pip
+  # jupyterhub/jupyterlab
   - nb_conda_kernels
-  # required to launch jupterlab from jupyterhub
-  - jupyterhub>=1.3.0
-  # jupyterhub menu https://github.com/jupyterlab/jupyterlab/issues/9428
-  - jupyterlab>=3.0.4
-  - ipywidgets>=7.6.0
-  - ipyparallel
-  # vscode
-  - code-server >= 3.2
-  # dask
-  - dask-gateway
-  - dask-labextension>=5.0.0
+  - ipython > 7
+  - jupyter-server-proxy
+  - jupyter_server==1.17.1
+  - jupyterlab==3.4.0
+  - jupyter_client
+  - jupyter_console
+  - jupyterhub==2.3.0
+  - nbconvert
+  - nbval
+  - retrolab
+
+  # jupyterlab extensions
+  - dask_labextension >= 5.3.0
+  - jupyterlab-git >=0.30.0
+  - sidecar >=0.5.0
+  - ipywidgets ==7.7.1
+  - ipyleaflet >=0.13.5
+  - pyviz_comms >=2.0.1
+  - jupyter-resource-usage >=0.6.0
+  - nbgitpuller
+  - jupyterlab_code_formatter
+
+  # cds dashboards
+  - cdsdashboards-singleuser==0.6.3
+  - jhsingle-native-proxy==0.8.0
+
+  # viz tools
+  - param
+  - python-graphviz
+  - plotly >=5.0
+
+  # testing, docs, linting
+  - pytest
+  - hypothesis
+  - flake8
+  - sphinx
+  - pytest-cov
+  - black
+  - isort
+  - importnb
+
   - pip:
-      # conda-store (mamba-org/gator)
-      - https://github.com/mamba-org/gator/archive/conda-store-integration.tar.gz
       # vscode jupyterlab launcher
       - git+https://github.com/betatim/vscode-binder
-      - batchspawner==1.1.0

roles/jupyterhub/templates/jupyterhub_config.py

@@ -53,12 +53,12 @@ def conda_envs_w_packages(packages, names_only=False):
 
 c.OAuthenticator.client_id = "{{ jupyterhub_client_id }}"
 c.OAuthenticator.client_secret = "{{ jupyterhub_client_secret }}"
-c.GenericOAuthenticator.oauth_callback_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}/hub/oauth_callback"
+c.GenericOAuthenticator.oauth_callback_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}/hub/oauth_callback"
 c.GenericOAuthenticator.scope = "profile"
 c.GenericOAuthenticator.tls_verify = False
-c.GenericOAuthenticator.authorize_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}/auth/realms/{{ keycloak_realm }}/protocol/openid-connect/auth"
-c.GenericOAuthenticator.token_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}/auth/realms/{{ keycloak_realm }}/protocol/openid-connect/token"
-c.GenericOAuthenticator.userdata_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}/auth/realms/{{ keycloak_realm }}/protocol/openid-connect/userinfo"
+c.GenericOAuthenticator.authorize_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}/auth/realms/{{ keycloak_realm }}/protocol/openid-connect/auth"
+c.GenericOAuthenticator.token_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}/auth/realms/{{ keycloak_realm }}/protocol/openid-connect/token"
+c.GenericOAuthenticator.userdata_url = "https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}/auth/realms/{{ keycloak_realm }}/protocol/openid-connect/userinfo"
 c.GenericOAuthenticator.username_key = "preferred_username"
 c.GenericOAuthenticator.claim_groups_key = "roles"
 c.GenericOAuthenticator.allowed_groups = ['jupyterhub_admin', 'jupyterhub_developer']
@@ -68,7 +68,7 @@ def conda_envs_w_packages(packages, names_only=False):
 def sync_users(username, min_uid=1_000_000, max_uid=1_000_000_000):
     current_uids = set()
     keycloak_admin = keycloak.KeycloakAdmin(
-        server_url="https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}/auth/",
+        server_url="https://{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}/auth/",
         username="{{ keycloak_admin_username }}",
         password="{{ keycloak_admin_password }}",
         realm_name="{{ keycloak_realm }}",
@@ -321,14 +321,14 @@ class QHubHPCSpawner(QHubHPCSpawnerBase):
 #==================== THEMING =====================#
 
 import tornado.web
-import qhub_jupyterhub_theme
+import nebari_jupyterhub_theme
 
 c.JupyterHub.extra_handlers = [
-    (r'/custom/(.*)', tornado.web.StaticFileHandler, {"path": qhub_jupyterhub_theme.STATIC_PATH}),
+    (r'/custom/(.*)', tornado.web.StaticFileHandler, {"path": nebari_jupyterhub_theme.STATIC_PATH}),
 ] + c.JupyterHub.extra_handlers
 
 c.JupyterHub.template_paths = [
-    qhub_jupyterhub_theme.TEMPLATE_PATH
+    nebari_jupyterhub_theme.TEMPLATE_PATH
 ] + c.JupyterHub.template_paths
 
 c.JupyterHub.template_vars = {

roles/traefik/templates/traefik_dynamic.yaml

@@ -88,33 +88,33 @@ http:
       service: "dummy"
 
     minio:
-      rule: "Host(`{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}`)"
+      rule: "Host(`{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}`)"
       entryPoints:
         - "minio"
       service: "minio"
     keycloak:
-      rule: "Host(`{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}`) && PathPrefix(`/auth`)"
+      rule: "Host(`{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}`) && PathPrefix(`/auth`)"
       entryPoints:
         - "websecure"
       service: "keycloak"
     jupyterhub:
-      rule: "Host(`{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}`) && PathPrefix(`{{ jupyterhub_base_url }}`)"
+      rule: "Host(`{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}`) && PathPrefix(`{{ jupyterhub_base_url }}`)"
       entryPoints:
         - "websecure"
       service: "jupyterhub"
     grafana:
-      rule: "Host(`{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}`) && PathPrefix(`{{ grafana_base_url }}`)"
+      rule: "Host(`{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}`) && PathPrefix(`{{ grafana_base_url }}`)"
       entryPoints:
         - "websecure"
       service: "grafana"
     dask-gateway:
-      rule: "Host(`{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}`) && (PathPrefix(`/clusters`) || PathPrefix(`/gateway`))"
+      rule: "Host(`{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}`) && (PathPrefix(`/clusters`) || PathPrefix(`/gateway`))"
       middlewares: ["dask-gateway-strip"]
       entryPoints:
         - "websecure"
       service: "dask-gateway"
     conda-store:
-      rule: "Host(`{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_host) }}`) && PathPrefix(`{{ conda_store_prefix }}`)"
+      rule: "Host(`{{ traefik_domain | default(hostvars[groups['hpc_master'][0]].ansible_ssh_host) }}`) && PathPrefix(`{{ conda_store_prefix }}`)"
       entryPoints:
         - "websecure"
       service: "conda-store"

tests/azure/main.tf

@@ -189,10 +189,10 @@ resource "local_file" "ansible_inventory" {
   all:
     hosts:
       ${azurerm_linux_virtual_machine.master-node.name}:
-        ansible_host: ${azurerm_linux_virtual_machine.master-node.public_ip_address} 
+        ansible_ssh_host: ${azurerm_linux_virtual_machine.master-node.public_ip_address}
         ansible_user: ubuntu
         ansible_ssh_private_key_file: ${var.ssh-private-key}
-      ${join("\n    ", formatlist("%s:\n      ansible_host: %s\n      ansible_user: ubuntu\n      ansible_ssh_private_key_file: ${var.ssh-private-key}\n      ansible_ssh_common_args: '-o ProxyCommand=\"ssh -i id_rsa -W %%h:%%p -q ubuntu@${azurerm_linux_virtual_machine.master-node.public_ip_address}\"'", azurerm_linux_virtual_machine.worker-nodes.*.name, azurerm_linux_virtual_machine.worker-nodes.*.private_ip_address))}
+      ${join("\n    ", formatlist("%s:\n      ansible_ssh_host: %s\n      ansible_user: ubuntu\n      ansible_ssh_private_key_file: ${var.ssh-private-key}\n      ansible_ssh_common_args: '-o ProxyCommand=\"ssh -i id_rsa -W %%h:%%p -q ubuntu@${azurerm_linux_virtual_machine.master-node.public_ip_address}\"'", azurerm_linux_virtual_machine.worker-nodes.*.name, azurerm_linux_virtual_machine.worker-nodes.*.private_ip_address))}
     children:
       hpc_master:
         hosts: