prep_import.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import CSVs into Neo4j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import shutil\n",
    "\n",
    "from glob import glob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download and unpack Neo4j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "version = \"neo4j-community-3.1.1\"\n",
    "name = \"neo4j/{}-unix.tar.gz\".format(version)\n",
    "url = \"http://neo4j.com/artifact.php?name={}-unix.tar.gz\".format(version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2017-03-07 19:40:12--  http://neo4j.com/artifact.php?name=neo4j-community-3.1.1-unix.tar.gz\n",
      "Resolving neo4j.com (neo4j.com)... 54.197.241.3, 54.243.71.145\n",
      "Connecting to neo4j.com (neo4j.com)|54.197.241.3|:80... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 77401077 (74M) [application/x-gzip]\n",
      "Saving to: ‘neo4j/neo4j-community-3.1.1-unix.tar.gz’\n",
      "\n",
      "neo4j/neo4j-communi 100%[===================>]  73.81M  87.4MB/s    in 0.8s    \n",
      "\n",
      "2017-03-07 19:40:13 (87.4 MB/s) - ‘neo4j/neo4j-community-3.1.1-unix.tar.gz’ saved [77401077/77401077]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget -O {name} {url}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "!tar -xf {name} -C neo4j"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Determine the names of the files we want to import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def replace_text(path, find, repl):\n",
    "    \"\"\"\n",
    "    Read a text file, replace the text specified by find with repl,\n",
    "    and overwrite the file with the modified version.\n",
    "    \"\"\"\n",
    "    with open(path) as read_file:\n",
    "        text = read_file.read()\n",
    "    pattern = re.escape(find)\n",
    "    text = re.sub(pattern, repl, text)\n",
    "    with open(path, 'wt') as write_file:\n",
    "        write_file.write(text)\n",
    "\n",
    "\n",
    "def create_instance(version, db_id, port):\n",
    "    \"\"\"Remove any old database files and prepare for new database.\"\"\"\n",
    "    floc = os.path.join(\"neo4j\", \"{}_{}\".format(version, db_id))\n",
    "    if os.path.exists(floc):\n",
    "        shutil.rmtree(floc)\n",
    "        \n",
    "    neo4j_loc = os.path.join(\"neo4j\", version)\n",
    "    shutil.copytree(neo4j_loc, floc)\n",
    "\n",
    "    # Modify neo4j-server.properties\n",
    "    path = os.path.join(floc, 'conf', 'neo4j.conf')\n",
    "\n",
    "    replace_text(path,\n",
    "        \"#dbms.security.auth_enabled=false\",\n",
    "        \"dbms.security.auth_enabled=false\"\n",
    "    )\n",
    "\n",
    "    replace_text(path,\n",
    "        \"#dbms.connector.bolt.listen_address=:7687\",\n",
    "        \"dbms.connector.bolt.listen_address=:{}\".format(port)\n",
    "    )\n",
    "\n",
    "    replace_text(path,\n",
    "        \"#dbms.connector.http.listen_address=:7474\",\n",
    "        \"dbms.connector.http.listen_address=:{}\".format(port+1)\n",
    "    )\n",
    "\n",
    "    replace_text(path,\n",
    "        \"dbms.connector.https.enabled=true\",\n",
    "        \"dbms.connector.https.enabled=false\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(\"../crossval_idx.txt\", \"r\") as fin:\n",
    "    crossval_idx = int(fin.read().strip())\n",
    "\n",
    "port_0 = 7500 + 100*crossval_idx\n",
    "\n",
    "db_name = \"rephetio-v2.0\"\n",
    "\n",
    "create_instance(version, db_name, port_0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['data/permuted/hetnet_perm-1.json.bz2', 'data/permuted/hetnet_perm-2.json.bz2', 'data/permuted/hetnet_perm-3.json.bz2', 'data/permuted/hetnet_perm-4.json.bz2', 'data/permuted/hetnet_perm-5.json.bz2']\n"
     ]
    }
   ],
   "source": [
    "perms = sorted(glob(\"data/permuted/hetnet_perm*\"))\n",
    "\n",
    "print(perms)\n",
    "\n",
    "for perm_idx, perm_net in enumerate(perms, start=1):\n",
    "    port = port_0 + 10*perm_idx\n",
    "    db_id = \"{}_perm-{}\".format(db_name, perm_idx)\n",
    "    create_instance(version, db_id, port)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!rm -rf neo4j/{version}"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}