basic_stats.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SI Collection Stats - stats.json generation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(`ujson` reads/writes faster, but escapes text improperly, so use `json` for final output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import json\n",
    "import re\n",
    "import ujson"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Key for variable naming:\n",
    "`stats` is the JSON object that will be output at the end\n",
    "\n",
    "We're looking for the following properties overall and per dept:\n",
    "* Count (total)\n",
    "* Departments\n",
    "* Country\n",
    "* On exhibit\n",
    "* Taxonomy (K,P only)\n",
    "* ~Type (general & specific)~ (TODO)\n",
    "* ~Public Domain~ (all meta is CC0 if here, should get total from collections.si.edu or look at has online media && online media is CC0)\n",
    "* ~Culture~ (TODO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "startTime = datetime.datetime.now()\n",
    "print(\"Started: \" + str(startTime))\n",
    "\n",
    "stepTime = datetime.datetime.now()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def printStepTime(text):\n",
    "    global stepTime\n",
    "    endTime = datetime.datetime.now()\n",
    "    print(\"Finished \" + text + \": \" + str(endTime) + \", took \" + str(endTime - stepTime))\n",
    "    stepTime = endTime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = {}\n",
    "stats[\"date\"] = datetime.datetime.now().timestamp()\n",
    "stats[\"total\"] = 0\n",
    "stats[\"depts\"] = {}\n",
    "#stats[\"type_general\"] = {}\n",
    "#stats[\"type_specific\"] = {}\n",
    "#stats[\"public_domain\"] = 0\n",
    "stats[\"on_exhibit\"] = 0\n",
    "stats[\"country\"] = {}\n",
    "stats[\"age\"] = {}\n",
    "stats[\"taxonomy\"] = {}\n",
    "stats[\"taxonomy\"][\"count\"] = 0\n",
    "stats[\"taxonomy\"][\"kingdom\"] = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "current_year = datetime.datetime.now().year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_countries = []\n",
    "with open('countries.json','r') as cin: # scraped from topojson data\n",
    "    valid_countries = json.load(cin)\n",
    "\n",
    "\n",
    "# Mapping of the following to topojson country names:\n",
    "# * Typos\n",
    "# * Other country name formattings (e.g. common vs historical vs formal vs topojson names)\n",
    "# * Abbreviations\n",
    "# * Accented letters\n",
    "# Assumes \"Islands\" has been replaced with \"Is.\" (since that fixes many topojson nuance cases)\n",
    "c_map = {\n",
    "    \"England\": \"United Kingdom\",\n",
    "    \"Britain\": \"United Kingdom\",\n",
    "    \"Great Britain\": \"United Kingdom\",\n",
    "    \"United States\": \"United States of America\",\n",
    "    \"US\": \"United States of America\",\n",
    "    \"USA\": \"United States of America\",\n",
    "    \"United Staes\": \"United States\",\n",
    "    'Antigua and Barbuda': 'Antigua and Barb.',\n",
    "    'Bolivia (Plurinational State of)': 'Bolivia',\n",
    "    'Bosnia and Herzegovina': 'Bosnia and Herz.',\n",
    "    'Brunei Darussalam': 'Brunei',\n",
    "    'Central African Republic': 'Central African Rep.',\n",
    "    \"Democratic People's Republic of Korea\": 'North Korea',\n",
    "    'Democratic Republic of the Congo': 'Dem. Rep. Congo',\n",
    "    'Dominican Republic': 'Dominican Rep.',\n",
    "    'Equatorial Guinea': 'Eq. Guinea',\n",
    "    'Iran (Islamic Republic of)': 'Iran',\n",
    "    \"Lao People's Democratic Republic\": 'Laos',\n",
    "    'Micronesia (Federated States of)': 'Micronesia',\n",
    "    'Republic of Korea': 'South Korea',\n",
    "    'Republic of Moldova': 'Moldova',\n",
    "    'Russian Federation': 'Russia',\n",
    "    'Saint Kitts and Nevis': 'St. Kitts and Nevis',\n",
    "    'Saint Vincent and the Grenadines': 'St. Vin. and Gren.',\n",
    "    'Sao Tome and Principe': 'São Tomé and Principe',\n",
    "    'South Sudan': 'S. Sudan',\n",
    "    'Swaziland': 'eSwatini',\n",
    "    'Syrian Arab Republic': 'Syria',\n",
    "    'The former Yugoslav Republic of Macedonia': 'Macedonia',\n",
    "    'United Republic of Tanzania': 'Tanzania',\n",
    "    'Venezuela (Bolivarian Republic of)': 'Venezuela',\n",
    "    'Viet Nam': 'Vietnam',\n",
    "    \"People's Republic of China\": \"China\",\n",
    "    \"Republic of the Congo\": \"Congo\",\n",
    "    \"Burma\": \"Myanmar\",\n",
    "    \"New Guinea\": \"Papua New Guinea\",\n",
    "    \"Colonial America\": \"United States of America\",\n",
    "    'Micronesia, Federated States of': 'Micronesia',\n",
    "    \"Czech Republic\": \"Czechia\",\n",
    "    \"Colonial America\": \"United States of America\",\n",
    "    \"French Canada\": \"Canada\",\n",
    "    \"U.S. Virgin Islands\": \"U.S. Virgin Is.\",\n",
    "    \"Cape Verde\": \"Cabo Verde\",\n",
    "    \"Curacao\": \"Curaçao\",\n",
    "    \"French Guiana\": \"Guyana\",\n",
    "    \"French Polynesia\": \"Fr. Polynesia\",\n",
    "    \"Northern Mariana Is.\": \"N. Mariana Is.\",\n",
    "    \"México\": \"Mexico\",\n",
    "    \"British Virgin Islands\": \"British Virgin Is.\",\n",
    "    \"Virgin Islands of the United States\": \"U.S. Virgin Is.\",\n",
    "    \"Congo, Democratic Rep. Of\": \"Dem. Rep. Congo\",\n",
    "    \"Burma (Myanmar)\": \"Myanmar\",\n",
    "    \"Federated States of Micronesia\": \"Micronesia\",\n",
    "    \"British Indian Ocean Territory\": \"Br. Indian Ocean Ter.\",\n",
    "    \"Cote d'Ivoire\": \"Côte d'Ivoire\",\n",
    "    \"Congo, Democratic Republic of\": \"Dem. Rep. Congo\",\n",
    "    \"Western Samoa\": \"Samoa\",\n",
    "    \"Belau\": \"Palau\",\n",
    "    \"Belau [Palau]\": \"Palau\",\n",
    "    \"Republic of Palau\": \"Palau\",\n",
    "    \"St. Lucia\": \"Saint Lucia\",\n",
    "    \"Ivory Coast\": \"Côte d'Ivoire\",\n",
    "    \"United States Minor Outlying Is.\": \"U.S. Minor Outlying Is.\",\n",
    "    \"Trinidad\": \"Trinidad and Tobago\",\n",
    "    \"Tobago\": \"Trinidad and Tobago\",\n",
    "    \"Republic of Ireland\": \"Ireland\",\n",
    "    \"Virgin Is. of the United States\": \"U.S. Virgin Is.\",\n",
    "    \"Northwestern Hawaiian Is.\": \"United States of America\",\n",
    "    \"Kingdom of Tonga\": \"Tonga\",\n",
    "    \"Republic of Fiji\": \"Fiji\",\n",
    "    \"Pitcairn Island\": \"Pitcairn Is.\",\n",
    "    \"West Germany\": \"Germany\",\n",
    "    \"Congo (Kinshasa)\": \"Congo\",\n",
    "    \"Main Hawaiian Is.\": \"United States of America\",\n",
    "    \"Holland\": \"Netherlands\",\n",
    "    \"Antigua\": \"Antigua and Barb.\",\n",
    "    \"St. Helena\": \"Saint Helena\",\n",
    "    \"Saint Helena Island\": \"Saint Helena\",\n",
    "    \"Hawaiian Is.\": \"United States of America\",\n",
    "    \"Myanmar [Burma]\": \"Myanmar\",\n",
    "    \"Kazakstan\": \"Kazakhstan\",\n",
    "    \"South Sandwich Is.\": \"S. Geo. and the Is.\",\n",
    "    \"Vanuatu (Republic of)\": \"Vanuatu\",\n",
    "    \"Tahiti\": \"Fr. Polynesia\",\n",
    "    \"Zaire\": \"Dem. Rep. Congo\",\n",
    "    \"Virgin Is. (U.S.)\": \"U.S. Virgin Is.\",\n",
    "    \"Johnston Atoll\": \"U.S. Minor Outlying Is.\",\n",
    "    \"Navassa Island\": \"U.S. Minor Outlying Is.\",\n",
    "    \"Sicily\": \"Italy\",\n",
    "    \"Etruria\": \"Italy\",\n",
    "}\n",
    "\n",
    "# Sanitizes country names\n",
    "# Checks valid topojson names\n",
    "# Does mappings for many common cases (see c_map above)\n",
    "# Checks case issues\n",
    "# Returns \"\" if cannot sanitize\n",
    "def sanitize_country(i):\n",
    "    if i == \"\":\n",
    "        return \"\"\n",
    "    \n",
    "    if i in valid_countries:\n",
    "        return i\n",
    "    \n",
    "    # remove entries with uncertainty and multiple origins\n",
    "    if \"?\" in i:\n",
    "        return \"\"\n",
    "    if \"/\" in i:\n",
    "        return \"\"\n",
    "    if \" or \" in i:\n",
    "        return \"\"\n",
    "    if \" and \" in i:\n",
    "        return \"\"\n",
    "    if \"not certain\" in i:\n",
    "        return \"\"\n",
    "    if \"Unknown\" in i:\n",
    "        return \"\"\n",
    "    if \"Not Stated\" in i:\n",
    "        return \"\"\n",
    "    if \"Not Given\" in i:\n",
    "        return \"\"\n",
    "    \n",
    "    # copy to local var\n",
    "    c = i\n",
    "    \n",
    "    # Remove regions\n",
    "    c = re.sub(r\"^now \",\"\",c)\n",
    "    c = re.sub(r\"^[Tt]he \",\"\",c)\n",
    "    c = c.replace(\"Islands\",\"Is.\")\n",
    "    \n",
    "    c_mapi = {}\n",
    "    for x in c_map:\n",
    "        c_mapi[x.lower()] = c_map[x]\n",
    "    for x in valid_countries:\n",
    "        c_mapi[x.lower()] = x\n",
    "        \n",
    "    if c.lower() in c_mapi:\n",
    "        return c_mapi[c.lower()]\n",
    "\n",
    "    # cannot sanitize\n",
    "    return \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t_k_map = {\n",
    "    'Eukaryota unassigned': 'Eukaryota',\n",
    "    'fungi': 'Fungi',\n",
    "    'Animalis': 'Animalia',\n",
    "    'Plantlist': 'Plantae',\n",
    "    'Planate': 'Plantae',\n",
    "    'Bacteria': 'Eubacteria',\n",
    "    'Eukarya': 'Eukaryota',\n",
    "    'Chromalveolata': 'Chromista',\n",
    "    'Liliales': 'Plantae',\n",
    "    'Metazoa': 'Animalia',\n",
    "    'Polygonales': 'Plantae',\n",
    "    'Protista': 'Protozoa',\n",
    "    'Protoctista': 'Protozoa',\n",
    "    'Rhizaria': 'Protozoa',\n",
    "}\n",
    "\n",
    "def sanitize_kingdom(i):\n",
    "    if i in t_k_map:\n",
    "        return t_k_map[i]\n",
    "    else:\n",
    "        return i\n",
    "\n",
    "t_p_blacklist = [\n",
    "    \"Incertae sedis\",\n",
    "    \"Unknown\",\n",
    "    \"Unidentified\",\n",
    "    \"Undetermined\",\n",
    "    \"Unsorted\",\n",
    "    \"unassigned\",\n",
    "    \"Indet.\",\n",
    "    \" or \",\n",
    "]\n",
    "\n",
    "def sanitize_phylum(i):\n",
    "    for b in t_p_blacklist:\n",
    "        if b.lower() in i.lower():\n",
    "            return \"Unknown\"\n",
    "    \n",
    "    return i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "printStepTime(\"setup\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Summarize\n",
    "# IMPORTANT! Remember to re-run the stats initialization cell or else you'll get duplicated data!\n",
    "with open('data/all_data.ndjson') as infile:\n",
    "    for line in infile:\n",
    "        data = ujson.loads(line)\n",
    "        \n",
    "        # Get object properties\n",
    "        unitCode = data['unitCode']\n",
    "        dataSource = \"Unknown\"\n",
    "        if 'data_source' in data['content']['descriptiveNonRepeating']:\n",
    "            dataSource = data['content']['descriptiveNonRepeating']['data_source']\n",
    "        elif unitCode == \"SI\":\n",
    "            # Fix \"SI\" unit b/c it's not consistent in reported name\n",
    "            dataSource = \"Smithsonian Institution, Digitization Program Office\"\n",
    "        else:\n",
    "            print(\"Unknown data source: \", data)\n",
    "    \n",
    "        #type_general = data['type']\n",
    "        #public_domain = False\n",
    "        #if \"online_media_rights\" in data['content']['indexedStructured']:\n",
    "        #    if data['content']['indexedStructured']['online_media_rights'] == 'No Restrictions':\n",
    "        #        public_domain = True\n",
    "        on_exhibit = False\n",
    "        country = \"\"\n",
    "        age = \"\"\n",
    "        \n",
    "        # Check b/c not all do -_-\n",
    "        if 'indexedStructured' in data['content']:\n",
    "            # Check if on exhibit\n",
    "            if \"onPhysicalExhibit\" in data['content']['indexedStructured']:\n",
    "                if data['content']['indexedStructured']['onPhysicalExhibit'] == \"Yes\":\n",
    "                    on_exhibit = True\n",
    "            # Check if we can get a country; prefer most accurate field (Matt Miller method)\n",
    "            if 'geoLocation' in data['content']['indexedStructured']:\n",
    "                for x in data['content']['indexedStructured']['geoLocation']:\n",
    "                    if 'L2' in x:\n",
    "                        if isinstance(x['L2'], str):\n",
    "                            country = sanitize_country(x['L2'])\n",
    "                        else:\n",
    "                            country = sanitize_country(x['L2']['content'])\n",
    "            # Check if we can get an age (most accurate level of detail is decade)\n",
    "            if \"date\" in data['content']['indexedStructured']:\n",
    "                for x in data['content']['indexedStructured']['date']:\n",
    "                    age_tmp = \"\"\n",
    "                    try:\n",
    "                        age_tmp = int(x.replace(\"BCE \", \"-\").replace(\"s\", \"\").replace(\"0-00\", \"0\"))\n",
    "                    except:\n",
    "                        print(\"Could not parse date: \", x)\n",
    "                    if age_tmp <= current_year: # sanity check for typos\n",
    "                        age = age_tmp\n",
    "            # Check for taxonomy data; this is on its own b/c it's across several units and so I'm tracking it as a special case\n",
    "            if \"tax_kingdom\" in data['content']['indexedStructured']:\n",
    "                # Kingdom; assume only one value in list\n",
    "                kingdom = sanitize_kingdom(data['content']['indexedStructured'][\"tax_kingdom\"][0])\n",
    "                \n",
    "                stats[\"taxonomy\"][\"count\"] += 1\n",
    "                \n",
    "                if not kingdom in stats[\"taxonomy\"][\"kingdom\"]:\n",
    "                    stats[\"taxonomy\"][\"kingdom\"][kingdom] = {}\n",
    "                    stats[\"taxonomy\"][\"kingdom\"][kingdom][\"count\"] = 0\n",
    "                    stats[\"taxonomy\"][\"kingdom\"][kingdom][\"phylum\"] = {}\n",
    "                \n",
    "                stats[\"taxonomy\"][\"kingdom\"][kingdom][\"count\"] += 1\n",
    "                \n",
    "                # Phylum; assume only one value in list\n",
    "                if \"tax_phylum\" in data['content']['indexedStructured']:\n",
    "                    phylum = sanitize_phylum(data['content']['indexedStructured'][\"tax_phylum\"][0])\n",
    "\n",
    "                    if not phylum in stats[\"taxonomy\"][\"kingdom\"][kingdom][\"phylum\"]:\n",
    "                        stats[\"taxonomy\"][\"kingdom\"][kingdom][\"phylum\"][phylum] = {}\n",
    "                        stats[\"taxonomy\"][\"kingdom\"][kingdom][\"phylum\"][phylum][\"count\"] = 0\n",
    "\n",
    "                    stats[\"taxonomy\"][\"kingdom\"][kingdom][\"phylum\"][phylum][\"count\"] += 1\n",
    "                else:\n",
    "                    if not \"Unknown\" in stats[\"taxonomy\"][\"kingdom\"][kingdom][\"phylum\"]:\n",
    "                        stats[\"taxonomy\"][\"kingdom\"][kingdom][\"phylum\"][\"Unknown\"] = {}\n",
    "                        stats[\"taxonomy\"][\"kingdom\"][kingdom][\"phylum\"][\"Unknown\"][\"count\"] = 0\n",
    "\n",
    "                    stats[\"taxonomy\"][\"kingdom\"][kingdom][\"phylum\"][\"Unknown\"][\"count\"] += 1\n",
    "        \n",
    "        # If department does not exist, create it\n",
    "        if not unitCode in stats[\"depts\"]:\n",
    "            stats[\"depts\"][unitCode] = {}\n",
    "            stats[\"depts\"][unitCode][\"count\"] = 0\n",
    "        #    stats[\"depts\"][unitCode][\"public_domain\"] = 0\n",
    "        #    stats[\"depts\"][unitCode][\"type_general\"] = {}\n",
    "            #stats[\"depts\"][unitCode][\"type_specific\"] = {}\n",
    "            stats[\"depts\"][unitCode][\"on_exhibit\"] = 0\n",
    "            stats[\"depts\"][unitCode][\"country\"] = {}\n",
    "            stats[\"depts\"][unitCode][\"age\"] = {}\n",
    "            stats[\"depts\"][unitCode][\"name\"] = dataSource\n",
    "            stats[\"depts\"][unitCode][\"unit_code\"] = unitCode\n",
    "        \n",
    "        # If general type does not exist, create it\n",
    "        #if not type_general in stats[\"type_general\"]:\n",
    "        #    stats[\"type_general\"][type_general] = {}\n",
    "        #    stats[\"type_general\"][type_general][\"count\"] = 0\n",
    "        #    stats[\"type_general\"][type_general][\"depts\"] = {}\n",
    "        #    stats[\"type_general\"][type_general][\"public_domain\"] = 0\n",
    "        \n",
    "        # If specific type does not exist, create it\n",
    "        # ...\n",
    "        \n",
    "        # Create cross-references one level deep\n",
    "        #if not type_general in stats[\"depts\"][dataSource][\"type_general\"]:\n",
    "        #    stats[\"depts\"][dataSource][\"type_general\"][type_general] = 0\n",
    "        #if not dataSource in stats[\"type_general\"][type_general][\"depts\"]:\n",
    "        #    stats[\"type_general\"][type_general][\"depts\"][dataSource] = 0\n",
    "        \n",
    "        # Add stats for this object\n",
    "        stats[\"total\"] += 1\n",
    "        stats[\"depts\"][unitCode][\"count\"] += 1\n",
    "        #stats[\"depts\"][dataSource][\"type_general\"][type_general] += 1\n",
    "        #stats[\"type_general\"][type_general][\"count\"] += 1\n",
    "        #stats[\"type_general\"][type_general][\"depts\"][dataSource] += 1\n",
    "        #if public_domain:\n",
    "        #    stats[\"public_domain\"] += 1\n",
    "        #    stats[\"depts\"][dataSource][\"public_domain\"] += 1\n",
    "        #    stats[\"type_general\"][type_general][\"public_domain\"] += 1\n",
    "        if on_exhibit:\n",
    "            stats[\"on_exhibit\"] += 1\n",
    "            stats[\"depts\"][unitCode][\"on_exhibit\"] += 1\n",
    "        if country != \"\":\n",
    "            if not country in stats[\"country\"]:\n",
    "                stats[\"country\"][country] = 0\n",
    "            if not country in stats[\"depts\"][unitCode][\"country\"]:\n",
    "                stats[\"depts\"][unitCode][\"country\"][country] = 0\n",
    "            stats[\"country\"][country] += 1\n",
    "            stats[\"depts\"][unitCode][\"country\"][country] += 1\n",
    "        if age != \"\":\n",
    "            if not age in stats[\"age\"]:\n",
    "                stats[\"age\"][age] = 0\n",
    "            if not age in stats[\"depts\"][unitCode][\"age\"]:\n",
    "                stats[\"depts\"][unitCode][\"age\"][age] = 0\n",
    "            stats[\"age\"][age] += 1\n",
    "            stats[\"depts\"][unitCode][\"age\"][age] += 1\n",
    "\n",
    "printStepTime(\"processing\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write stats to file\n",
    "with open('docs/stats.json','w') as out:\n",
    "    out.write(json.dumps(stats))\n",
    "    #out.write(json.dumps(stats, indent=4)) # Pretty print"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}