From 2a252e99bd6d54030672c521a69638fe0dfb0d22 Mon Sep 17 00:00:00 2001 From: Lance-Drane Date: Wed, 17 Jul 2024 17:27:26 -0400 Subject: [PATCH] add Jupyter URL update endpoint Signed-off-by: Lance-Drane --- docker-compose-complete.yml | 54 ++++++++++++++++++++++ docker-compose.yml | 43 +++-------------- ipsportal/api.py | 2 +- ipsportal/data_api.py | 82 +++++++++++++++++++++++++++++++-- ipsportal/templates/events.html | 19 +++++++- 5 files changed, 157 insertions(+), 43 deletions(-) create mode 100644 docker-compose-complete.yml diff --git a/docker-compose-complete.yml b/docker-compose-complete.yml new file mode 100644 index 0000000..305c2d2 --- /dev/null +++ b/docker-compose-complete.yml @@ -0,0 +1,54 @@ +# full software stack + +version: '3' +services: + db: + image: mongo:6 + restart: unless-stopped + + app: + build: + context: . + restart: unless-stopped + environment: + MONGO_HOST: db + MINIO_PRIVATE_URL: http://minio:9000 + MINIO_PUBLIC_URL: http://localhost/files + JAEGER_HOST: jaeger + depends_on: + - db + - minio + + jaeger: + image: jaegertracing/all-in-one:1.38 + environment: + COLLECTOR_ZIPKIN_HOST_PORT: 9411 + QUERY_BASE_PATH: /jaeger + + web: + image: nginx + ports: + - 80:80 + volumes: + - ./nginx.conf:/etc/nginx/conf.d/default.conf + depends_on: + - app + - jaeger + + mongo-express: + image: mongo-express + ports: + - 8081:8081 + environment: + ME_CONFIG_MONGODB_SERVER: db + depends_on: + - db + + minio: + image: "bitnami/minio:2024.6.4" + environment: + # references: https://github.com/bitnami/containers/blob/main/bitnami/minio/README.md + MINIO_ROOT_USER: AKIAIOSFODNN7EXAMPLE + MINIO_ROOT_PASSWORD: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + #volumes: + #- "./tmp/minio:/bitnami/minio/data" diff --git a/docker-compose.yml b/docker-compose.yml index 0428de0..f8972a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,52 +1,21 @@ +# minimal needed while testing out the application + version: '3' services: db: image: mongo:6 restart: unless-stopped - - app: - build: - context: . - restart: unless-stopped - environment: - MONGO_HOST: db - MINIO_PRIVATE_URL: http://minio:9000 - MINIO_PUBLIC_URL: http://localhost/files - JAEGER_HOST: jaeger - depends_on: - - db - - minio - - jaeger: - image: jaegertracing/all-in-one:1.38 - environment: - COLLECTOR_ZIPKIN_HOST_PORT: 9411 - QUERY_BASE_PATH: /jaeger - - web: - image: nginx ports: - - 80:80 - volumes: - - ./nginx.conf:/etc/nginx/conf.d/default.conf - depends_on: - - app - - jaeger + - 27017:27017 - mongo-express: - image: mongo-express - ports: - - 8081:8081 - environment: - ME_CONFIG_MONGODB_SERVER: db - depends_on: - - db - minio: image: "bitnami/minio:2024.6.4" environment: # references: https://github.com/bitnami/containers/blob/main/bitnami/minio/README.md MINIO_ROOT_USER: AKIAIOSFODNN7EXAMPLE MINIO_ROOT_PASSWORD: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + ports: + - 9000:9000 + - 9001:9001 #volumes: #- "./tmp/minio:/bitnami/minio/data" diff --git a/ipsportal/api.py b/ipsportal/api.py index 801b277..f816095 100644 --- a/ipsportal/api.py +++ b/ipsportal/api.py @@ -126,7 +126,7 @@ def trace(portal_runid: str) -> Tuple[Response, int]: @bp.route("/", methods=['POST']) @bp.route("/api/event", methods=['POST']) def event() -> Tuple[Response, int]: - event_list: Optional[Union[List[Dict[str, Any]], Dict[str, Any]]] = request.get_json() + event_list: Optional[Union[List[Dict[str, Any]], Dict[str, Any]]] = request.get_json() # type: ignore[attr-defined] if event_list is None: current_app.logger.error("Missing data") diff --git a/ipsportal/data_api.py b/ipsportal/data_api.py index f04611a..0df67de 100644 --- a/ipsportal/data_api.py +++ b/ipsportal/data_api.py @@ -13,7 +13,10 @@ @bp.route("/api/data/runs") def data_runs() -> Tuple[Response, int]: - runs = [run["portal_runid"] for run in db.data.find(projection={"_id": False, "portal_runid": True})] + runs = [ + run["portal_runid"] + for run in db.data.find(projection={"_id": False, "portal_runid": True}) + ] return jsonify(runs), 200 @@ -106,6 +109,14 @@ def add() -> Tuple[Response, int]: if not portal_runid: return jsonify("Missing value for HTTP Header X-Ips-Portal-Runid"), 400 + # Jupyter links are optional to associate with a data event. + # If they DO exist, we use a non-printable delimiter to separate links in the header value + juypter_links_header = request.headers.get("X-Ips-Jupyter-Links") + if juypter_links_header: + jupyter_links = juypter_links_header.split("\x01") + else: + jupyter_links = [] + # runid is needed because the portal_runid is not a valid bucket name for MINIO runid = get_runid(portal_runid) if runid is None: @@ -132,13 +143,78 @@ def add() -> Tuple[Response, int]: db.data.update_one( {"portal_runid": portal_runid, "runid": runid}, { - "$push": {"tags": {"tag": resolved_tag, "data_location_url": data_location_url}}, + "$push": { + "tags": { + "tag": resolved_tag, + "data_location_url": data_location_url, + "jupyter_urls": jupyter_links, + } + }, }, upsert=True, ) return jsonify(data_location_url), 201 +@bp.route("/api/data/add_url", methods=["PUT"]) +def add_url() -> Tuple[Response, int]: + """ + PUT Jupyter URL links + + Response body should be JSON, i.e. + { + "url": "https://jupyterlink.com", + "tags": [ + "1.1231", + "2324.124" + ] + } + + The return value will be an array of data URL locations (JSONified). The response codes are: + - 200 - created + - 400 - portal_runid didn't exist, or structure of data was wrong + - 500 - server error + """ + + data = request.get_json() # type: ignore[attr-defined] + errors = [] + url = data.get('url') + if not isinstance(url, str): + errors.append({"url": "Must be provided and a string"}) + + if not isinstance(data.get('tags'), list): + errors.append({"tags": "Must be provided and a non-empty list"}) + else: + try: + tag_lookups = set(map(float, data.get('tags'))) + except ValueError: + errors.append({"tags": "Must be floating point values"}) + + portal_runid = data.get('portal_runid') + if not isinstance(portal_runid, str): + errors.append({"portal_runid": "Must be provided"}) + else: + result = db.data.find_one( + {"portal_runid": portal_runid}, + ) + if not result: + errors.append({"portal_runid": f"{portal_runid} does not exist"}) + + if errors: + return jsonify(errors), 400 + + # TODO figure out how to do this entirely in Mongo + for tags_prop in result['tags']: # type: ignore[index] + if tags_prop['tag'] in tag_lookups and url not in tags_prop['jupyter_urls']: + tags_prop['jupyter_urls'].append(url) + db.data.replace_one( + {'_id': result['_id']}, # type: ignore[index] + result # type: ignore[arg-type] + ) + + return jsonify([]), 200 + + @bp.route("/api/data/query", methods=["POST"]) def query() -> Tuple[Response, int]: return ( @@ -146,7 +222,7 @@ def query() -> Tuple[Response, int]: sorted( x["portal_runid"] for x in db.data.find( - request.get_json(), projection={"_id": False, "portal_runid": True} + request.get_json(), projection={"_id": False, "portal_runid": True} # type: ignore[attr-defined] ) ) ), diff --git a/ipsportal/templates/events.html b/ipsportal/templates/events.html index b0c7977..afe1199 100644 --- a/ipsportal/templates/events.html +++ b/ipsportal/templates/events.html @@ -105,10 +105,25 @@

{% block title %}Run - {{ run.runid }}{% endblock %}

{% if data_info is not none %}
- Raw Data Files + Raw Data
    {% for item in data_info %} -
  1. {{item.tag}}
    2. +
  2. +
    + {{item.tag}} +

    Download raw data

    + {% if item.jupyter_urls %} +
    + Associated JupyterHub Links +
      + {% for url in item.jupyter_urls %} +
    • {{url}}
      • + {% endfor %} +
    +
    + {% endif %} +
    +
    3. {% endfor %}