add Jupyter URL update endpoint

Signed-off-by: Lance-Drane <ldraneutk@gmail.com>

docker-compose-complete.yml

@@ -0,0 +1,54 @@
+# full software stack
+
+version: '3'
+services:
+  db:
+    image: mongo:6
+    restart: unless-stopped
+
+  app:
+    build:
+      context: .
+    restart: unless-stopped
+    environment:
+      MONGO_HOST: db
+      MINIO_PRIVATE_URL: http://minio:9000
+      MINIO_PUBLIC_URL: http://localhost/files
+      JAEGER_HOST: jaeger
+    depends_on:
+     - db
+     - minio
+
+  jaeger:
+    image: jaegertracing/all-in-one:1.38
+    environment:
+      COLLECTOR_ZIPKIN_HOST_PORT: 9411
+      QUERY_BASE_PATH: /jaeger
+
+  web:
+    image: nginx
+    ports:
+      - 80:80
+    volumes:
+      - ./nginx.conf:/etc/nginx/conf.d/default.conf
+    depends_on:
+      - app
+      - jaeger
+
+  mongo-express:
+    image: mongo-express
+    ports:
+      - 8081:8081
+    environment:
+      ME_CONFIG_MONGODB_SERVER: db
+    depends_on:
+     - db
+
+  minio:
+    image: "bitnami/minio:2024.6.4"
+    environment:
+      # references: https://github.com/bitnami/containers/blob/main/bitnami/minio/README.md
+      MINIO_ROOT_USER: AKIAIOSFODNN7EXAMPLE
+      MINIO_ROOT_PASSWORD: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+    #volumes:
+      #- "./tmp/minio:/bitnami/minio/data"

docker-compose.yml

@@ -1,52 +1,21 @@
+# minimal needed while testing out the application
+
 version: '3'
 services:
   db:
     image: mongo:6
     restart: unless-stopped
-
-  app:
-    build:
-      context: .
-    restart: unless-stopped
-    environment:
-      MONGO_HOST: db
-      MINIO_PRIVATE_URL: http://minio:9000
-      MINIO_PUBLIC_URL: http://localhost/files
-      JAEGER_HOST: jaeger
-    depends_on:
-     - db
-     - minio
-
-  jaeger:
-    image: jaegertracing/all-in-one:1.38
-    environment:
-      COLLECTOR_ZIPKIN_HOST_PORT: 9411
-      QUERY_BASE_PATH: /jaeger
-
-  web:
-    image: nginx
     ports:
-      - 80:80
-    volumes:
-      - ./nginx.conf:/etc/nginx/conf.d/default.conf
-    depends_on:
-      - app
-      - jaeger
+      - 27017:27017
 
-  mongo-express:
-    image: mongo-express
-    ports:
-      - 8081:8081
-    environment:
-      ME_CONFIG_MONGODB_SERVER: db
-    depends_on:
-     - db
-
   minio:
     image: "bitnami/minio:2024.6.4"
     environment:
       # references: https://github.com/bitnami/containers/blob/main/bitnami/minio/README.md
       MINIO_ROOT_USER: AKIAIOSFODNN7EXAMPLE
       MINIO_ROOT_PASSWORD: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+    ports:
+      - 9000:9000
+      - 9001:9001
     #volumes:
       #- "./tmp/minio:/bitnami/minio/data"

ipsportal/api.py

@@ -126,7 +126,7 @@ def trace(portal_runid: str) -> Tuple[Response, int]:
 @bp.route("/", methods=['POST'])
 @bp.route("/api/event", methods=['POST'])
 def event() -> Tuple[Response, int]:
-    event_list: Optional[Union[List[Dict[str, Any]], Dict[str, Any]]] = request.get_json()
+    event_list: Optional[Union[List[Dict[str, Any]], Dict[str, Any]]] = request.get_json()  # type: ignore[attr-defined]
 
     if event_list is None:
         current_app.logger.error("Missing data")

ipsportal/data_api.py

@@ -13,7 +13,10 @@
 
 @bp.route("/api/data/runs")
 def data_runs() -> Tuple[Response, int]:
-    runs = [run["portal_runid"] for run in db.data.find(projection={"_id": False, "portal_runid": True})]
+    runs = [
+        run["portal_runid"]
+        for run in db.data.find(projection={"_id": False, "portal_runid": True})
+    ]
     return jsonify(runs), 200
 
 
@@ -106,6 +109,14 @@ def add() -> Tuple[Response, int]:
     if not portal_runid:
         return jsonify("Missing value for HTTP Header X-Ips-Portal-Runid"), 400
 
+    # Jupyter links are optional to associate with a data event.
+    # If they DO exist, we use a non-printable delimiter to separate links in the header value
+    juypter_links_header = request.headers.get("X-Ips-Jupyter-Links")
+    if juypter_links_header:
+        jupyter_links = juypter_links_header.split("\x01")
+    else:
+        jupyter_links = []
+
     # runid is needed because the portal_runid is not a valid bucket name for MINIO
     runid = get_runid(portal_runid)
     if runid is None:
@@ -132,21 +143,86 @@ def add() -> Tuple[Response, int]:
     db.data.update_one(
         {"portal_runid": portal_runid, "runid": runid},
         {
-            "$push": {"tags": {"tag": resolved_tag, "data_location_url": data_location_url}},
+            "$push": {
+                "tags": {
+                    "tag": resolved_tag,
+                    "data_location_url": data_location_url,
+                    "jupyter_urls": jupyter_links,
+                }
+            },
         },
         upsert=True,
     )
     return jsonify(data_location_url), 201
 
 
+@bp.route("/api/data/add_url", methods=["PUT"])
+def add_url() -> Tuple[Response, int]:
+    """
+    PUT Jupyter URL links
+
+    Response body should be JSON, i.e.
+    {
+      "url": "https://jupyterlink.com",
+      "tags": [
+        "1.1231",
+        "2324.124"
+      ]
+    }
+
+    The return value will be an array of data URL locations (JSONified). The response codes are:
+    - 200 - created
+    - 400 - portal_runid didn't exist, or structure of data was wrong
+    - 500 - server error
+    """
+
+    data = request.get_json()  # type: ignore[attr-defined]
+    errors = []
+    url = data.get('url')
+    if not isinstance(url, str):
+        errors.append({"url": "Must be provided and a string"})
+
+    if not isinstance(data.get('tags'), list):
+        errors.append({"tags": "Must be provided and a non-empty list"})
+    else:
+        try:
+            tag_lookups = set(map(float, data.get('tags')))
+        except ValueError:
+            errors.append({"tags": "Must be floating point values"})
+
+    portal_runid = data.get('portal_runid')
+    if not isinstance(portal_runid, str):
+        errors.append({"portal_runid": "Must be provided"})
+    else:
+        result = db.data.find_one(
+            {"portal_runid": portal_runid},
+        )
+        if not result:
+            errors.append({"portal_runid": f"{portal_runid} does not exist"})
+
+    if errors:
+        return jsonify(errors), 400
+
+    # TODO figure out how to do this entirely in Mongo
+    for tags_prop in result['tags']:  # type: ignore[index]
+        if tags_prop['tag'] in tag_lookups and url not in tags_prop['jupyter_urls']:
+            tags_prop['jupyter_urls'].append(url)
+    db.data.replace_one(
+        {'_id': result['_id']},  # type: ignore[index]
+        result  # type: ignore[arg-type]
+    )
+
+    return jsonify([]), 200
+
+
 @bp.route("/api/data/query", methods=["POST"])
 def query() -> Tuple[Response, int]:
     return (
         jsonify(
             sorted(
                 x["portal_runid"]
                 for x in db.data.find(
-                    request.get_json(), projection={"_id": False, "portal_runid": True}
+                    request.get_json(), projection={"_id": False, "portal_runid": True}  # type: ignore[attr-defined]
                 )
             )
         ),

ipsportal/templates/events.html

@@ -105,10 +105,25 @@ <h4>{% block title %}Run - {{ run.runid }}{% endblock %}</h4>
 
 {% if data_info is not none %}
 <details>
-  <summary>Raw Data Files</summary>
+  <summary>Raw Data</summary>
   <ol>
   {% for item in data_info %}
-  <li><a class="raw-data-uri" href="{{item.data_location_url}}" download="{{run.portal_runid}}__{{item.tag}}">{{item.tag}}</a></li>
+  <li>
+    <details>
+      <summary>{{item.tag}}</summary>
+      <p><a class="raw-data-uri" href="{{item.data_location_url}}" download="{{run.portal_runid}}__{{item.tag}}">Download raw data</a></p>
+      {% if item.jupyter_urls %}
+      <details>
+        <summary>Associated JupyterHub Links</summary>
+        <ul>
+        {% for url in item.jupyter_urls %}
+          <li><a href="{{url}}">{{url}}</a></li>
+        {% endfor %}
+        </ul>
+      </details>  
+      {% endif %}    
+    </details>
+  </li>
   {% endfor %}
   </ol>
 </details>