{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Leaflet cluster map of talk locations\n",
    "\n",
    "Assuming you are working in a Linux or Windows Subsystem for Linux environment, you may need to install some dependencies. Assuming a clean installation, the following will be needed:\n",
    "\n",
    "```bash\n",
    "sudo apt install jupyter\n",
    "sudo apt install python3-pip\n",
    "pip install python-frontmatter getorg --upgrade\n",
    "```\n",
    "\n",
    "After which you can run this from the `_talks/` directory, via:\n",
    "\n",
    "```bash\n",
    " jupyter nbconvert --to notebook --execute talkmap.ipynb --output talkmap_out.ipynb\n",
    "```\n",
    " \n",
    "The `_talks/` directory contains `.md` files of all your talks. This scrapes the location YAML field from each `.md` file, geolocates it with `geopy/Nominatim`, and uses the `getorg` library to output data, HTML, and Javascript for a standalone cluster map."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-02-26T03:36:06.769536Z",
     "iopub.status.busy": "2025-02-26T03:36:06.769131Z",
     "iopub.status.idle": "2025-02-26T03:36:07.934020Z",
     "shell.execute_reply": "2025-02-26T03:36:07.933257Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting python-frontmatter\r\n",
      "  Downloading python_frontmatter-1.1.0-py3-none-any.whl.metadata (4.1 kB)\r\n",
      "Requirement already satisfied: getorg in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (0.3.1)\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: PyYAML in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (from python-frontmatter) (6.0.2)\r\n",
      "Requirement already satisfied: geopy in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (from getorg) (2.4.1)\r\n",
      "Requirement already satisfied: pygithub in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (from getorg) (2.6.1)\r\n",
      "Requirement already satisfied: retrying in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (from getorg) (1.3.4)\r\n",
      "Requirement already satisfied: geographiclib<3,>=1.52 in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (from geopy->getorg) (2.0)\r\n",
      "Requirement already satisfied: pynacl>=1.4.0 in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (from pygithub->getorg) (1.5.0)\r\n",
      "Requirement already satisfied: requests>=2.14.0 in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (from pygithub->getorg) (2.32.3)\r\n",
      "Requirement already satisfied: pyjwt>=2.4.0 in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (from pyjwt[crypto]>=2.4.0->pygithub->getorg) (2.10.1)\r\n",
      "Requirement already satisfied: typing-extensions>=4.0.0 in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (from pygithub->getorg) (4.12.2)\r\n",
      "Requirement already satisfied: urllib3>=1.26.0 in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (from pygithub->getorg) (2.3.0)\r\n",
      "Requirement already satisfied: Deprecated in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (from pygithub->getorg) (1.2.18)\r\n",
      "Requirement already satisfied: six>=1.7.0 in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (from retrying->getorg) (1.17.0)\r\n",
      "Requirement already satisfied: cryptography>=3.4.0 in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (from pyjwt[crypto]>=2.4.0->pygithub->getorg) (44.0.1)\r\n",
      "Requirement already satisfied: cffi>=1.4.1 in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (from pynacl>=1.4.0->pygithub->getorg) (1.17.1)\r\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (from requests>=2.14.0->pygithub->getorg) (3.4.1)\r\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (from requests>=2.14.0->pygithub->getorg) (3.10)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (from requests>=2.14.0->pygithub->getorg) (2025.1.31)\r\n",
      "Requirement already satisfied: wrapt<2,>=1.10 in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (from Deprecated->pygithub->getorg) (1.17.2)\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pycparser in /opt/hostedtoolcache/Python/3.9.21/x64/lib/python3.9/site-packages (from cffi>=1.4.1->pynacl>=1.4.0->pygithub->getorg) (2.22)\r\n",
      "Downloading python_frontmatter-1.1.0-py3-none-any.whl (9.8 kB)\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Installing collected packages: python-frontmatter\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully installed python-frontmatter-1.1.0\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iywidgets and ipyleaflet support disabled. You must be in a Jupyter notebook to use this feature.\n",
      "Error raised:\n",
      "No module named 'ipyleaflet'\n",
      "Check that you have enabled ipyleaflet in Jupyter with:\n",
      "    jupyter nbextension enable --py ipyleaflet\n"
     ]
    }
   ],
   "source": [
    "# Start by installing the dependencies\n",
    "!pip install python-frontmatter getorg --upgrade\n",
    "import frontmatter\n",
    "import glob\n",
    "import getorg\n",
    "from geopy import Nominatim\n",
    "from geopy.exc import GeocoderTimedOut"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-02-26T03:36:07.936149Z",
     "iopub.status.busy": "2025-02-26T03:36:07.935901Z",
     "iopub.status.idle": "2025-02-26T03:36:07.939314Z",
     "shell.execute_reply": "2025-02-26T03:36:07.938765Z"
    }
   },
   "outputs": [],
   "source": [
    "# Collect the Markdown files\n",
    "g = glob.glob(\"_talks/*.md\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "execution": {
     "iopub.execute_input": "2025-02-26T03:36:07.941580Z",
     "iopub.status.busy": "2025-02-26T03:36:07.941049Z",
     "iopub.status.idle": "2025-02-26T03:36:07.966143Z",
     "shell.execute_reply": "2025-02-26T03:36:07.965594Z"
    }
   },
   "outputs": [],
   "source": [
    "# Set the default timeout, in seconds\n",
    "TIMEOUT = 5\n",
    "\n",
    "# Prepare to geolocate\n",
    "geocoder = Nominatim(user_agent=\"academicpages.github.io\")\n",
    "location_dict = {}\n",
    "location = \"\"\n",
    "permalink = \"\"\n",
    "title = \"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the event that this times out with an error, double check to make sure that the location is can be properly geolocated."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-02-26T03:36:07.968167Z",
     "iopub.status.busy": "2025-02-26T03:36:07.967976Z",
     "iopub.status.idle": "2025-02-26T03:36:11.603488Z",
     "shell.execute_reply": "2025-02-26T03:36:11.602744Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Talk 2 on Relevant Topic in Your Field<br />London School of Testing; London, UK London, Greater London, England, United Kingdom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tutorial 1 on Relevant Topic in Your Field<br />UC-Berkeley Institute for Testing Science; Berkeley, CA, USA Berkeley, Alameda County, California, United States\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Conference Proceeding talk 3 on Relevant Topic in Your Field<br />Testing Institute of America 2014 Annual Conference; Los Angeles, CA, USA Los Angeles, Los Angeles County, California, United States\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Talk 1 on Relevant Topic in Your Field<br />UC San Francisco, Department of Testing; San Francisco, CA, USA San Francisco, California, United States\n"
     ]
    }
   ],
   "source": [
    "# Perform geolocation\n",
    "for file in g:\n",
    "    # Read the file\n",
    "    data = frontmatter.load(file)\n",
    "    data = data.to_dict()\n",
    "\n",
    "    # Press on if the location is not present\n",
    "    if 'location' not in data:\n",
    "        continue\n",
    "\n",
    "    # Prepare the description\n",
    "    title = data['title'].strip()\n",
    "    venue = data['venue'].strip()\n",
    "    location = data['location'].strip()\n",
    "    description = f\"{title}<br />{venue}; {location}\"\n",
    "\n",
    "    # Geocode the location and report the status\n",
    "    try:\n",
    "        location_dict[description] = geocoder.geocode(location, timeout=TIMEOUT)\n",
    "        print(description, location_dict[description])\n",
    "    except ValueError as ex:\n",
    "        print(f\"Error: geocode failed on input {location} with message {ex}\")\n",
    "    except GeocoderTimedOut as ex:\n",
    "        print(f\"Error: geocode timed out on input {location} with message {ex}\")\n",
    "    except Exception as ex:\n",
    "        print(f\"An unhandled exception occurred while processing input {location} with message {ex}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-02-26T03:36:11.605607Z",
     "iopub.status.busy": "2025-02-26T03:36:11.605411Z",
     "iopub.status.idle": "2025-02-26T03:36:11.613528Z",
     "shell.execute_reply": "2025-02-26T03:36:11.612932Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Written map to talkmap/'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save the map\n",
    "m = getorg.orgmap.create_map_obj()\n",
    "getorg.orgmap.output_html_cluster_map(location_dict, folder_name=\"talkmap\", hashed_usernames=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.21"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}