run_HISAT2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import shutil\n",
    "import os\n",
    "import tarfile\n",
    "import stat\n",
    "import multiprocessing as mp\n",
    "import subprocess\n",
    "import sys"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch_project_df(profID):\n",
    "  '''Wrapper to API at www.ebi.ac.uk/ena'''\n",
    "  import requests\n",
    "  import json\n",
    "  import pandas\n",
    "\n",
    "  url = 'https://www.ebi.ac.uk/ena/portal/api/filereport'\n",
    "  params = {'accession':profID,\n",
    "            'format':'json',\n",
    "            'download':'false',\n",
    "            'result':'read_run'}\n",
    "  headers = {\"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64)\"+\n",
    "             \" AppleWebKit/537.36 (KHTML, like Gecko)\"+\n",
    "             \" Chrome/70.0.3538.77 Safari/537.36\"}\n",
    "\n",
    "  print(\"Accessing: \", url)\n",
    "  r = requests.get(url,\n",
    "                  params = params,\n",
    "                  headers = headers\n",
    "                  ) \n",
    "  print(\"\\tMaking Request To: \", r.url) \n",
    "  my_json = json.loads(r.text)\n",
    "  df = pandas.json_normalize(json.loads(r.text))\n",
    "  return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_hisat2(histast_url):\n",
    "  import time\n",
    "  import shutil\n",
    "  import os\n",
    "  import tarfile\n",
    "  import stat\n",
    "\n",
    "  installed_wget = True\n",
    "  try:\n",
    "    import wget\n",
    "  except ModuleNotFoundError:\n",
    "    installed_wget = False\n",
    "    import subprocess\n",
    "\n",
    "  \n",
    "  print(\"Downloading HISAT2 at: \", histast_url)\n",
    "  if installed_wget: \n",
    "    wget.download(histast_url)\n",
    "  else:\n",
    "    subprocess.run([\"wget\", histast_url])\n",
    "\n",
    "  print(\"Inflating HISAT2 at: \", os.getcwd())\n",
    "  shutil.unpack_archive(histast_url.split(\"/\")[-1], os.getcwd(),\"zip\")\n",
    "  \n",
    "  #for root, dirs, files in os.walk(os.path.join(os.getcwd(),\n",
    "  #          histast_url.split(\"/\")[-1].replace(\"-Linux_x86_64.zip\",\"\"))):\n",
    "  #  for d in dirs:\n",
    "  #    os.chmod(os.path.join(root, d), stat.S_IEXEC) #owner execute\n",
    "  #  for f in files:\n",
    "  #    os.chmod(os.path.join(root, f), stat.S_IEXEC) #owner execute\n",
    "\n",
    "  hisatpath = os.path.join(os.getcwd(),\n",
    "            histast_url.split(\"/\")[-1].replace(\"-Linux_x86_64.zip\",\"\"),\n",
    "             \"hisat2\")\n",
    "  return hisatpath\n",
    "\n",
    "def download_indexes(index_url):\n",
    "  import time\n",
    "  import shutil\n",
    "  import os\n",
    "  import tarfile\n",
    "  import stat\n",
    "\n",
    "  installed_wget = True\n",
    "  try:\n",
    "    import wget\n",
    "  except ModuleNotFoundError:\n",
    "    installed_wget = False\n",
    "    import subprocess\n",
    "\n",
    "  print(\"Downloading INDEXES at: \", index_url)\n",
    "  if installed_wget: \n",
    "    wget.download(index_url)\n",
    "  else:\n",
    "    subprocess.run([\"wget\", index_url])\n",
    "\n",
    "  print(\"Inflating Indexes at: \", os.getcwd())\n",
    "  file = tarfile.open(index_url.split(\"/\")[-1])\n",
    "  file.extractall('{}/{}'.format(os.getcwd(),\n",
    "                                 index_url.split(\"/\")[-1].replace(\".tar.gz\",\"\")))\n",
    "\n",
    "  indexpath = os.path.join(os.getcwd(),\n",
    "            index_url.split(\"/\")[-1].replace(\".tar.gz\",\"\"))\n",
    "  return indexpath"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Install and Build hisat and index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_local_path = \"/gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/\"\n",
    "datapath = \"/gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Main"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HISAT2 Path: /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/hisat2-2.2.1/hisat2\n",
      "Indexes Path: /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/grch38_genome\n"
     ]
    }
   ],
   "source": [
    "# Path to HISAT2 executable\n",
    "HISAT2_path = ''.join([my_local_path,\"hisat2-2.2.1/hisat2\"])\n",
    "\n",
    "# Path to reference genomes\n",
    "index_path = ''.join([my_local_path,\"grch38_genome\"])\n",
    "\n",
    "print(\"HISAT2 Path: {}\".format(HISAT2_path))\n",
    "print(\"Indexes Path: {}\".format(index_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(HISAT2_path):\n",
    "    hisat2_url= \"https://cloud.biohpc.swmed.edu/index.php/s/oTtGWbWjaxsQ2Ho/download/hisat2-2.2.1-Linux_x86_64.zip\"\n",
    "    HISAT2_path = build_hisat2(hisat2_url)\n",
    "    \n",
    "if not os.path.exists(index_path):\n",
    "    index_url = \"https://genome-idx.s3.amazonaws.com/hisat/grch38_genome.tar.gz\"\n",
    "    index_path = download_indexes(index_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bam_files  PRJEB35615\tPRJNA281708  PRJNA634526\n",
      "csv_files  PRJNA277616\tPRJNA41223   sam_files\n"
     ]
    }
   ],
   "source": [
    "!ls /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing Project: PRJNA281708\n",
      "Accessing:  https://www.ebi.ac.uk/ena/portal/api/filereport\n",
      "\tMaking Request To:  https://www.ebi.ac.uk/ena/portal/api/filereport?accession=PRJNA281708&format=json&download=false&result=read_run\n",
      "\tProject Path:  /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/PRJNA281708\n",
      "\n",
      "\tRun:  SRR1982814\n",
      "\tSingle Read? : True\n",
      "SRR1982814 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA281708/SRR1982814.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  SRR1982612\n",
      "\tSingle Read? : True\n",
      "SRR1982612 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA281708/SRR1982612.sam Exists. Continuing.\n",
      "\n",
      "\n",
      "\tRun:  SRR1982783\n",
      "\tSingle Read? : True\n",
      "SRR1982783 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA281708/SRR1982783.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  SRR1982660\n",
      "\tSingle Read? : True\n",
      "SRR1982660 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA281708/SRR1982660.sam Exists. Continuing.\n",
      "Processing Project: PRJNA634526\n",
      "Accessing:  https://www.ebi.ac.uk/ena/portal/api/filereport\n",
      "\tMaking Request To:  https://www.ebi.ac.uk/ena/portal/api/filereport?accession=PRJNA634526&format=json&download=false&result=read_run\n",
      "\tProject Path:  /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/PRJNA634526\n",
      "\n",
      "\tRun:  SRR14209603\n",
      "\tSingle Read? : False\n",
      "SRR14209603 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA634526/SRR14209603.sam Exists. Continuing.\n",
      "\n",
      "\n",
      "\tRun:  SRR14209634\n",
      "\tSingle Read? : False\n",
      "SRR14209634 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA634526/SRR14209634.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  SRR14209655\n",
      "\tSingle Read? : False\n",
      "SRR14209655 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA634526/SRR14209655.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  SRR14209591\n",
      "\tSingle Read? : False\n",
      "SRR14209591 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA634526/SRR14209591.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  SRR14209592\n",
      "\tSingle Read? : False\n",
      "SRR14209592 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA634526/SRR14209592.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  SRR14209589\n",
      "\tSingle Read? : False\n",
      "SRR14209589 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA634526/SRR14209589.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  SRR14209708\n",
      "\tSingle Read? : False\n",
      "SRR14209708 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA634526/SRR14209708.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  SRR14209614\n",
      "\tSingle Read? : False\n",
      "SRR14209614 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA634526/SRR14209614.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  SRR14209590\n",
      "\tSingle Read? : False\n",
      "SRR14209590 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA634526/SRR14209590.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  SRR14209651\n",
      "\tSingle Read? : False\n",
      "SRR14209651 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA634526/SRR14209651.sam Exists. Continuing.\n",
      "Processing Project: PRJEB35615\n",
      "Accessing:  https://www.ebi.ac.uk/ena/portal/api/filereport\n",
      "\tMaking Request To:  https://www.ebi.ac.uk/ena/portal/api/filereport?accession=PRJEB35615&format=json&download=false&result=read_run\n",
      "\tProject Path:  /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/PRJEB35615\n",
      "\n",
      "\tRun:  ERR3698089\n",
      "\tSingle Read? : False\n",
      "ERR3698089 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJEB35615/ERR3698089.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  ERR3698079\n",
      "\tSingle Read? : False\n",
      "ERR3698079 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJEB35615/ERR3698079.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  ERR3698058\n",
      "\tSingle Read? : False\n",
      "ERR3698058 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJEB35615/ERR3698058.sam Exists. Continuing.\n",
      "\n",
      "\n",
      "\tRun:  ERR3698061\n",
      "\tSingle Read? : False\n",
      "ERR3698061 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJEB35615/ERR3698061.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  ERR3698080\n",
      "\tSingle Read? : False\n",
      "ERR3698080 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJEB35615/ERR3698080.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  ERR3698084\n",
      "\tSingle Read? : False\n",
      "ERR3698084 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJEB35615/ERR3698084.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  ERR3698055\n",
      "\tSingle Read? : False\n",
      "ERR3698055 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJEB35615/ERR3698055.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  ERR3698060\n",
      "\tSingle Read? : False\n",
      "ERR3698060 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJEB35615/ERR3698060.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  ERR3698057\n",
      "\tSingle Read? : False\n",
      "ERR3698057 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJEB35615/ERR3698057.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  ERR3698053\n",
      "\tSingle Read? : False\n",
      "ERR3698053 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJEB35615/ERR3698053.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  ERR3698056\n",
      "\tSingle Read? : False\n",
      "ERR3698056 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJEB35615/ERR3698056.sam Exists. Continuing.\n",
      "Processing Project: PRJNA277616\n",
      "Accessing:  https://www.ebi.ac.uk/ena/portal/api/filereport\n",
      "\tMaking Request To:  https://www.ebi.ac.uk/ena/portal/api/filereport?accession=PRJNA277616&format=json&download=false&result=read_run\n",
      "\tProject Path:  /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/PRJNA277616\n",
      "\n",
      "\tRun:  SRR1867792\n",
      "\tSingle Read? : False\n",
      "SRR1867792 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA277616/SRR1867792.sam Exists. Continuing.\n",
      "\n",
      "\n",
      "\tRun:  SRR1909639\n",
      "\tSingle Read? : False\n",
      "SRR1909639 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA277616/SRR1909639.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  SRR1926136\n",
      "\tSingle Read? : False\n",
      "SRR1926136 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA277616/SRR1926136.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  SRR1926132\n",
      "\tSingle Read? : False\n",
      "SRR1926132 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA277616/SRR1926132.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  SRR1909613\n",
      "\tSingle Read? : False\n",
      "SRR1909613 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA277616/SRR1909613.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  SRR1926135\n",
      "\tSingle Read? : False\n",
      "SRR1926135 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA277616/SRR1926135.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  SRR1926134\n",
      "\tSingle Read? : False\n",
      "SRR1926134 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA277616/SRR1926134.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  SRR1926133\n",
      "\tSingle Read? : False\n",
      "SRR1926133 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA277616/SRR1926133.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  SRR1909638\n",
      "\tSingle Read? : False\n",
      "SRR1909638 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA277616/SRR1909638.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  SRR1909637\n",
      "\tSingle Read? : False\n",
      "SRR1909637 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA277616/SRR1909637.sam Exists. Continuing.\n",
      "Processing Project: PRJNA41223\n",
      "Accessing:  https://www.ebi.ac.uk/ena/portal/api/filereport\n",
      "\tMaking Request To:  https://www.ebi.ac.uk/ena/portal/api/filereport?accession=PRJNA41223&format=json&download=false&result=read_run\n",
      "\tProject Path:  /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/PRJNA41223\n",
      "\n",
      "\tRun:  ERR015526\n",
      "\tSingle Read? : False\n",
      "ERR015526 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA41223/ERR015526.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  ERR013145\n",
      "\tSingle Read? : False\n",
      "ERR013145 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA41223/ERR013145.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  ERR013144\n",
      "\tSingle Read? : False\n",
      "ERR013144 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA41223/ERR013144.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  ERR015530\n",
      "\tSingle Read? : False\n",
      "ERR015530 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA41223/ERR015530.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  ERR015969\n",
      "\tSingle Read? : False\n",
      "ERR015969 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA41223/ERR015969.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  ERR015879\n",
      "\tSingle Read? : False\n",
      "ERR015879 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA41223/ERR015879.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  ERR013159\n",
      "\tSingle Read? : False\n",
      "ERR013159 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA41223/ERR013159.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  ERR016004\n",
      "\tSingle Read? : False\n",
      "ERR016004 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA41223/ERR016004.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  ERR016160\n",
      "\tSingle Read? : False\n",
      "ERR016160 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA41223/ERR016160.sam Exists. Continuing.\n",
      "\n",
      "\tRun:  ERR013151\n",
      "\tSingle Read? : False\n",
      "ERR013151 already is processsed. /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/sam_files/PRJNA41223/ERR013151.sam Exists. Continuing.\n"
     ]
    }
   ],
   "source": [
    "directories = os.listdir(datapath)\n",
    "#directories = [\"PRJNA475246\",\"PRJEB35615\"] # Process Single Project\n",
    "for directory in directories:\n",
    "    if not directory.startswith(\"PRJ\"):\n",
    "        continue\n",
    "    print(\"Processing Project: {}\".format(directory))\n",
    "    # Fetch Metadata for Project from ENA API\n",
    "    df = fetch_project_df(directory)\n",
    "    project = '/'.join([datapath, directory])\n",
    "    print(\"\\tProject Path: \",project)\n",
    "    \n",
    "    runs = [x.replace(\".fastq\",\"\").replace(\"_1\",\"\").replace(\"_2\",\"\") for x in sorted(os.listdir(project))]\n",
    "    runs = list(set(runs))\n",
    "    single_read = False\n",
    "    for i in range(len(runs)):\n",
    "        print()\n",
    "        run = runs[i]\n",
    "        #print(\"\\tRun: \", run)\n",
    "        outfile = \"{}.sam\".format(run)\n",
    "        subset = df[df['run_accession']==run]\n",
    "        if subset.empty: continue\n",
    "        if ';' in subset['fastq_ftp'].values[0]:\n",
    "            urls = subset['fastq_ftp'].values[0].split(\";\")\n",
    "        else:\n",
    "            urls = [subset['fastq_ftp'].values[0]]\n",
    "            single_read = True\n",
    "        print(\"\\tRun: \",run)\n",
    "        print(\"\\tSingle Read? :\", single_read)\n",
    "        \n",
    "        \n",
    "        # Dont Reprocesses if the .sam file exists \n",
    "        if os.path.exists('{}/sam_files/{}/{}'.format(datapath, directory,outfile)):\n",
    "            print(\"{} already is processsed. {} Exists. Continuing.\".format(run,\n",
    "                                    '{}/sam_files/{}/{}'.format(datapath, directory,outfile)))\n",
    "            continue\n",
    "        if single_read:\n",
    "            read1 = os.path.join(project, \"{}.fastq\".format(run))\n",
    "            if not os.path.exists(read1):\n",
    "                read1 = os.path.join(project, \"{}_1.fastq\".format(run))   \n",
    "            cmd = [HISAT2_path,\n",
    "           \"-p{}\".format(mp.cpu_count()),\n",
    "           \"-q\",\n",
    "           \"-x\",\n",
    "           \"$HISAT2_INDEXES/genome\",\n",
    "           \"-U\",\n",
    "           read1,\n",
    "           \"-S\",\n",
    "           outfile,\n",
    "           \"--time\"]\n",
    "        else:\n",
    "            read1 = os.path.join(project, \"{}_1.fastq\".format(run))\n",
    "            read2 = os.path.join(project, \"{}_2.fastq\".format(run))\n",
    "\n",
    "            cmd = [HISAT2_path,\n",
    "           \"-p{}\".format(mp.cpu_count()),\n",
    "           \"-q\",\n",
    "           \"-x\",\n",
    "           \"$HISAT2_INDEXES/genome\",\n",
    "           \"-1\",\n",
    "           read1,\n",
    "           \"-2\",\n",
    "           read2,\n",
    "           \"-S\",\n",
    "           outfile,\n",
    "           \"--time\"]\n",
    "        print(\"\\tRunning Command: \")\n",
    "        print(*cmd, sep =\" \")\n",
    "        my_env = {**os.environ,\n",
    "          'HISAT2_INDEXES': '{}/grch38/'.format(index_path)}\n",
    "\n",
    "        process = subprocess.Popen(cmd, \n",
    "                            stdout=subprocess.PIPE, \n",
    "                            stderr=subprocess.STDOUT,\n",
    "                            env = my_env)\n",
    "        for line in process.stdout:\n",
    "            sys.stdout.write(line)\n",
    "        print()\n",
    "        print(\"Moving {} to {}\".format(outfile, '{}/sam_files/{}/{}'.format(datapath, directory,outfile)))\n",
    "        \n",
    "        if not os.path.exists('{}/sam_files/{}'.format(datapath, directory)):\n",
    "            os.makedirs('{}/sam_files/{}'.format(datapath, directory))\n",
    "        shutil.move(outfile, '{}/sam_files/{}/{}'.format(datapath, directory,outfile))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}