Merge pull request #330 from MetaCell/release/0.7.0

Release/0.7.0

.dockerignore

@@ -2,18 +2,25 @@ org.geppetto.frontend.jupyter
 pygeppetto
 .vscode
 .ipynb_checkpoints
-netpyne_ui/geppetto
+
 *.egg-info
 notebook.ipynb
 .git
-netpyne
+
 Dockerfile
 Dockerfile_base
 Dockerfile_dev
-netpyne_workspace
-webapp/node_modules
+node_modules
 webapp/build
 webapp/geppetto-client
+workspace
+netpyne_workspace
+.idea
 k8s
-src
-x86_64
+/src
+x86_64
+app.log
+/.pytest_cache
+.jupyter-config
+**/*.pyc
+**/__pycache__

.gitignore

@@ -13,11 +13,15 @@ init.py
 webapp/node_modules
 webapp/geppetto-client
 webapp/build
+workspace
 netpyne_workspace
+tests/frontend/e2e/node_modules
 Dockerfile_mini
 npm*
 .vscode
 app.log
 utilities/x86_64
 .idea
+*.iml
 x86_64
+.jupyter-config

.travis.yml

@@ -4,6 +4,7 @@ python:
   - "3.7"
 node_js:
   - "12"
+cache: pip
 env:
   global:
     - secure: dn0FPQ5IG4M/3kdwnyI78ElQ308Vc3QnKAvkWfwMFb8QxDqxQdnTo7AV1qTMtbLrDNkeEWIgi4nc7jmXNtvGTwOfhAULVh6606Qs5B+ezTdwzajbbFMI8SKQx/pnTojOMu8dx7V4lMoR/YWcojR0VC1IWVC62TGbSB1k5BDGgH0=
@@ -14,13 +15,13 @@ notifications:
 install:
   - sudo apt-get install libgnutls28-dev
   - sudo apt install libcurl4-openssl-dev libssl-dev
-  - sudo apt-get install python3-dev  
+  - sudo apt-get install python3-dev
 script:
-
-  - python utilities/install.py 
+  - python utilities/install.py
   - ./NetPyNE-UI &
   - http_status=$(curl -s -I $1 $LANDING_PAGE | grep HTTP/1.1 | awk {'print $2'})
   - echo "$http_status"
-  - cd webapp
-  - travis_retry npm run test -- --verbose --colors
+  - cd tests/frontend/e2e
+  - npm ci
+  - sleep 10 && travis_retry npm run test
   - exit 0

Dockerfile

@@ -1,17 +1,33 @@
 FROM frodriguez4600/jupyter-neuron:v7.8.0
+ARG INSTALLATION_FOLDER=/home/jovyan/work/NetPyNE-UI
+ARG NETPYNE_VERSION=development
+ARG WORKSPACE_VERSION=nov2020
+ARG JUPYTER_GEPPETTO_VERSION=development
+ARG PYGEPPETTO_VERSION=development
+ARG BUILD_ARGS=""
+
 USER $NB_USER
 
-ARG branch=development 
-RUN echo "$branch";
+ENV INSTALLATION_FOLDER=$INSTALLATION_FOLDER
+ENV NETPYNE_VERSION=$NETPYNE_VERSION
+ENV WORKSPACE_VERSION=$WORKSPACE_VERSION
+ENV JUPYTER_GEPPETTO_VERSION=$JUPYTER_GEPPETTO_VERSION
+ENV PYGEPPETTO_VERSION=$PYGEPPETTO_VERSION
+ENV BUILD_ARGS=$BUILD_ARGS
 
-ENV INSTALLATION_FOLDER=/home/jovyan/work/NetPyNE-UI
 WORKDIR /home/jovyan/work
+COPY --chown=1000:1000 requirements.txt ${INSTALLATION_FOLDER}/requirements.txt
+
+WORKDIR ${INSTALLATION_FOLDER}
+RUN pip install -r requirements.txt
 
-COPY --chown=1000:1000 . NetPyNE-UI
+COPY --chown=1000:1000 . .
 WORKDIR ${INSTALLATION_FOLDER}/utilities
 
-RUN python install.py branch $branch
+RUN python install.py ${BUILD_ARGS}
 
 WORKDIR ${INSTALLATION_FOLDER}
 
+RUN pip install -r requirements-test.txt
+RUN pytest tests/backend
 CMD /bin/bash -c "jupyter notebook --NotebookApp.default_url=/geppetto --NotebookApp.token='' --library=netpyne_ui --NotebookApp.disable_check_xsrf=True"

NetPyNE-UI

@@ -1,2 +1,4 @@
 #!/bin/sh
+CONFIG=$(pwd)/.jupyter-config
+export JUPYTER_CONFIG_DIR=$CONFIG
 exec jupyter notebook --NotebookApp.default_url=/geppetto --NotebookApp.token='' --library=netpyne_ui --NotebookApp.disable_check_xsrf=True

README.md

@@ -42,19 +42,77 @@ conda activate netpyne
 ### Run install script
 
 ```bash
-git clone https://github.com/MetaCell/NetPyNE-UI.git
-cd utilities
-python install.py
-cd ..
+python utilities/install.py
+```
+
+### Start application
+
+```bash
 ./NetPyNE-UI
 ```
 
-### To update sources
+For debugging you can use `run.py` instead
 
 ```bash
-python update.py
+python run.py
 ```
 
-NetPyNE-UI is being developed in collaboration with the [Neurosim Lab](http://neurosimlab.org/).
-See the [Wiki](https://github.com/MetaCell/NetPyNE-UI/wiki) for more info!
 
+## Run NetPyNE User Interface in Docker
+
+Ensure that you have Docker installed on your system.
+
+Build the image
+
+```bash
+docker build -t netpyne-ui
+```
+
+Run the image
+
+```bash
+docker run -p 8888:8888 netpyne-ui 
+```
+
+## End-to-end tests
+
+End-to-end tests are located in `tests/deployment/frontend/e2e`. 
+Ensure that the application is running in a blank state, since end-to-end tests interact with the running application.
+
+Install packages
+
+```bash
+cd tests/frontend/e2e
+npm install
+```
+
+Start tests
+
+```bash
+npm run test
+```
+
+
+#### Containerized tests
+
+You can also use `docker-compose` to run the tests. 
+Ensure that you have Docker installed on your system.
+
+Build the images
+
+```bash
+cd tests/deployment
+sh build.sh
+```
+
+Run the tests
+
+```bash
+docker-compose up --abort-on-container-exit --exit-code-from netpyne-ui-e2e
+```
+
+
+## Additional Notes
+
+NetPyNE-UI is being developed in collaboration with the [Neurosim Lab](http://neurosimlab.org/).
+See the [Wiki](https://github.com/MetaCell/NetPyNE-UI/wiki) for more info!

jupyter_hub/Dockerfile_spawner

@@ -21,7 +21,7 @@ RUN /bin/bash -c "source activate snakes && python --version"
 RUN /bin/bash -c "source activate snakes && exec python install.py branch $netpyneuiBranch"
 WORKDIR /home/jovyan
 RUN git clone https://github.com/Neurosim-lab/netpyne_workspace
-WORKDIR /home/jovyan/netpyne_workspace
+WORKDIR /home/jovyan/workspace
 RUN ln -sfn /home/jovyan/work/NetPyNE-UI-$netpyneuiBranch/netpyne_ui/tests tests
 
 # Copy jupyterhub_config

k8s/cf_pipeline.yaml

@@ -10,41 +10,33 @@ steps:
     repo: "MetaCell/NetPyNE-UI"
     revision: "${{CF_BRANCH}}"
     stage: "clone"
-    when:
-      branch:
-        only:
-          - "${{CF_BRANCH}}"
   BuildingNetPyNE-UI:
     title: Building NetPyNE-UI
     type: build
     stage: build
     image_name: netpyne-ui
     working_directory: ${{main_clone}}
-    tag: '${{CF_SHORT_REVISION}}'
+    tag: '${{CF_BRANCH_TAG_NORMALIZED}}-${{CF_SHORT_REVISION}}-${{CF_BUILD_TIMESTAMP}}'
     build_arguments:
-      - branch=${{CF_BRANCH}}
+      - NETPYNE_VERSION=${{NETPYNE_VERSION}}
+      - WORKSPACE_VERSION=${{WORKSPACE_VERSION}}
+      - PYGEPPETTO_VERSION=${{PYGEPPETTO_VERSION}}
+      - JUPYTER_GEPPETTO_VERSION=${{JUPYTER_GEPPETTO_VERSION}}
+      - BUILD_ARGS=${{BUILD_ARGS}}
     registry: ${{REGISTRY}}
-    when:
-      branch:
-        only:
-          - "${{CF_BRANCH}}"
   BuildingHub:
     title: Building Hub
     type: build
     stage: "build"
     image_name: netpyne-hub
     working_directory: k8s
-    tag: '${{CF_SHORT_REVISION}}'
+    tag: '${{CF_BRANCH_TAG_NORMALIZED}}-${{CF_SHORT_REVISION}}-${{CF_BUILD_TIMESTAMP}}'
     registry: ${{REGISTRY}}
-    when:
-      branch:
-        only:
-          - "${{CF_BRANCH}}"
   installing_chart:
     type: helm
     stage: deploy
     title: "Deploy chart"
-    working_directory: NetPyNE-UI/k8s
+    working_directory: k8s
     arguments:
       action: install
       chart_name: jupyterhub
@@ -57,6 +49,8 @@ steps:
       custom_value_files: 
         - 'cf_values.yaml'
       custom_values:
-        - 'hub_image_tag=${{CF_SHORT_REVISION}}'
-        - 'singleuser_image_tag=${{CF_SHORT_REVISION}}'
-        - 'proxy_secretToken=${{SECRET_TOKEN}}'
+        - 'hub_image_tag=${{CF_BRANCH_TAG_NORMALIZED}}-${{CF_SHORT_REVISION}}-${{CF_BUILD_TIMESTAMP}}'
+        - 'singleuser_image_tag=${{CF_BRANCH_TAG_NORMALIZED}}-${{CF_SHORT_REVISION}}-${{CF_BUILD_TIMESTAMP}}'
+        - 'singleuser_memory_limit=${{MEMORY_MAX}}'
+        - 'singleuser_memory_guarantee=${{MEMORY_MIN}}'
+        - 'proxy_secretToken=${{SECRET_TOKEN}}'

k8s/cf_values.yaml

@@ -34,7 +34,11 @@ auth:
 singleuser:
   storage:
     type: none
+  cpu:
+    limit: 0.5
+    guarantee: 0.2
   memory:
+    limit: 0.5G
     guarantee: 0.2G
   image:
     name:  gcr.io/metacellllc/netpyne-ui

netpyne_ui/constants.py

@@ -1,15 +1,25 @@
 import os
 
 UPLOAD_FOLDER_NAME = 'uploads'
-NETPYNE_WORKDIR = 'netpyne_workspace'
+NETPYNE_WORKDIR = 'workspace'
 
 ALLOWED_EXTENSIONS = ["py", "zip", "gz", ".tar.gz", "pdf", "txt", "xls", "png", "jpeg", "hoc"]
+HERE = os.path.dirname(os.path.abspath(__file__))
+ROOT = os.path.dirname(HERE)
+UPLOAD_FOLDER_PATH = os.path.join(ROOT, NETPYNE_WORKDIR, UPLOAD_FOLDER_NAME)
+NETPYNE_WORKDIR_PATH = os.path.join(ROOT, NETPYNE_WORKDIR)
 
-UPLOAD_FOLDER_PATH = os.path.join(os.getcwd(), NETPYNE_WORKDIR, UPLOAD_FOLDER_NAME)
-NETPYNE_WORKDIR_PATH = os.path.join(os.getcwd(), NETPYNE_WORKDIR)
+if not os.path.exists(NETPYNE_WORKDIR_PATH):
+    NETPYNE_WORKDIR_PATH = os.path.join(os.getcwd(), NETPYNE_WORKDIR)
+
+if not os.path.exists(NETPYNE_WORKDIR_PATH):
+    raise Exception(f"Workdir path {NETPYNE_WORKDIR_PATH} does not exist")
 
 if not os.path.exists(UPLOAD_FOLDER_PATH):
     os.makedirs(UPLOAD_FOLDER_PATH)
 
 if not os.path.exists(NETPYNE_WORKDIR_PATH):
-    os.makedirs(NETPYNE_WORKDIR_PATH)
+    os.makedirs(NETPYNE_WORKDIR_PATH)
+
+# Number of connections above this limit are considered too many to be shown.
+NUM_CONN_LIMIT = 1000