EMBOSS_doc_parse_pyacd.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import yaml\n",
    "import sys\n",
    "import struct\n",
    "\n",
    "from lxml import etree\n",
    "\n",
    "parser = etree.HTMLParser()\n",
    "#qualifiers_page = etree.parse('http://emboss.open-bio.org/html/use/apcs02.html', parser)\n",
    "#attributes_resp = etree.parse('http://emboss.open-bio.org/html/dev/apas05.html', parser)\n",
    "datatypes_page = etree.parse('http://emboss.sourceforge.net/developers/acd/syntax.html', parser)\n",
    "datatype = ''\n",
    "maxint = 2 ** (struct.Struct('i').size * 8 - 1) - 1\n",
    "TYPES = {\n",
    "    'integer':int,\n",
    "    'Y/N': bool,\n",
    "    'string':str,\n",
    "    'float': float,\n",
    "    'boolean':bool\n",
    "}\n",
    "VALUES = {\n",
    "    '0':0,\n",
    "    '1':1,\n",
    "    'N':False,\n",
    "    'Y':True,\n",
    "    '\"\"':'',\n",
    "    '':'',\n",
    "    'FLT_MAX':sys.float_info.max,\n",
    "    '-FLT_MAX':-sys.float_info.max,\n",
    "    'INT_MIN':maxint - 1,\n",
    "    'INT_MAX':maxint,    \n",
    "}\n",
    "\n",
    "datatypes_reference = {}\n",
    "\n",
    "#parse datatypes and descriptions and input/output\n",
    "for line in datatypes_page.xpath('//a[@name=\"sect23\"]/following-sibling::table/tr[td[not(@colspan)]/p/b/text()]'):\n",
    "    datatype_res = line.xpath('td/p/b/text()')\n",
    "    if len(datatype_res)!=1:\n",
    "        continue\n",
    "    datatype = str(datatype_res[0])\n",
    "    # guess if the datatype is an input and output by parsing the previous \"subsection\" table line...\n",
    "    if line.xpath('./preceding-sibling::tr[td/@colspan=\"5\"][1]/td/p/b/text()')[0] in [\"Output types\", \"Graphics types\"]:\n",
    "        parameter_type = \"OUTPUT\"\n",
    "    else:\n",
    "        parameter_type = \"INPUT\"\n",
    "    description = str(line.xpath('td[2]/p/text()')[0])\n",
    "    datatypes_reference[datatype] = {'description': description,'qualifiers':{},'attributes':{}, 'type': parameter_type}\n",
    "\n",
    "# parse qualifiers\n",
    "datatype = \"\"\n",
    "for line in datatypes_page.xpath('//a[@name=\"sect2531\" or @name=\"sect2532\" or @name=\"sect2533\"]/following-sibling::table[1]/tr[not(td/p/b)]'):\n",
    "    datatype_res = line.xpath('td[1]/p/text()')\n",
    "    if len(datatype_res)!=1:\n",
    "        continue\n",
    "    datatype = str(datatype_res[0].strip()) if str(datatype_res[0].strip())!='' else datatype\n",
    "    qualifier_list = line.xpath('./td[2]/p/text()')\n",
    "    qualifier = qualifier_list[0].strip()[1:-1]\n",
    "    value_type_list = line.xpath('./td[2]/p/i/text()')\n",
    "    value_type = TYPES[value_type_list[0]]\n",
    "    description_default_list = line.xpath('./td[3]/p/text()')\n",
    "    description = description_default_list[0]\n",
    "    default_value = VALUES[description_default_list[1].replace('Default: ','').strip()]\n",
    "    datatypes_reference[datatype]['qualifiers'][qualifier]={\n",
    "        'value_type':str(value_type().__class__.__name__),\n",
    "        'description':str(description),\n",
    "        'default_value':default_value\n",
    "    }\n",
    "\n",
    "# parse attributes    \n",
    "datatype = \"\"\n",
    "for line in datatypes_page.xpath('////h5[starts-with(text(),\"2.4.1.2.1\") or starts-with(text(),\"2.4.1.2.2\") or starts-with(text(),\"2.4.1.2.3\") or starts-with(text(),\"2.4.1.2.4\") or starts-with(text(),\"2.4.1.2.5\")]/following-sibling::table[1]/tr[not(td/p/b)]'):\n",
    "    datatype_res = line.xpath('td[1]/p/text()')\n",
    "    if len(datatype_res)!=1:\n",
    "        continue\n",
    "    datatype = str(datatype_res[0].strip()) if str(datatype_res[0].strip())!='' else datatype\n",
    "    attribute_list = line.xpath('./td[2]/p/text()')\n",
    "    attribute = attribute_list[0].strip()[:-1]\n",
    "    value_type_list = line.xpath('./td[2]/p/i/text()')\n",
    "    value_type = TYPES[value_type_list[0]]\n",
    "    description_default_list = line.xpath('./td[3]/p//text()')\n",
    "    description = description_default_list[0]\n",
    "    if (len(description_default_list)>2):\n",
    "        pre_value = description_default_list[2][1:-1]\n",
    "    else:\n",
    "        pre_value = description_default_list[1].replace('Default: ','').strip()\n",
    "    default_value = VALUES.get(pre_value)\n",
    "    if default_value is None:\n",
    "        default_value = value_type(pre_value)\n",
    "    #print(datatype, attribute, value_type().__class__.__name__)\n",
    "    datatypes_reference[datatype]['attributes'][attribute] = {\n",
    "        'default_value': default_value,\n",
    "        'value_type': str(value_type().__class__.__name__),\n",
    "        'description': str(description)\n",
    "    }\n",
    "with open('datatypes_reference.yml', 'w') as outfile:\n",
    "    yaml.safe_dump(datatypes_reference, outfile)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}