process_fda_approved_names.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "from gensim.models import KeyedVectors\n",
    "wv = KeyedVectors.load_word2vec_format('bio_embedding_intrinsic', binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dict'>\n",
      "False\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "# trying to see how I might filter for drug names\n",
    "# especially those with multi-word names\n",
    "print(type(wv.vocab))\n",
    "print('butabarbital-sodium' in wv.vocab.keys())\n",
    "print('butabarbital' in wv.vocab.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9019\n",
      "8-hour bayer\n",
      "8-mop\n",
      "a-hydrocort\n",
      "a-methapred\n",
      "a-poxide\n",
      "a.p.l.\n",
      "a/t/s\n",
      "abacavir and lamivudine\n",
      "abacavir sulfate\n",
      "abacavir sulfate and lamivudine\n",
      "abacavir sulfate, lamivudine and zidovudine\n",
      "abacavir sulfate; dolutegravir sodium; lamivudine\n",
      "abacavir sulfate; lamivudine\n",
      "abacavir sulfate; lamivudine; zidovudine\n",
      "abacavir sulfate;lamivudine\n",
      "abacavir; lamivudine\n",
      "abacavir;lamivudine\n",
      "abaloparatide\n",
      "abarelix\n",
      "abatacept\n",
      "abciximab\n",
      "abelcet\n",
      "abemaciclib\n",
      "abilify\n",
      "abilify maintena kit\n",
      "abilify mycite kit\n",
      "abiraterone acetate\n",
      "abitrexate\n",
      "ablavar\n",
      "ablysinol\n",
      "abobotulinumtoxina\n",
      "abraxane\n",
      "abreva\n",
      "abrilada\n",
      "absorica\n",
      "absorica ld\n",
      "abstral\n",
      "acalabrutinib\n",
      "acamprosate calcium\n",
      "acanya\n",
      "acarbose\n",
      "accolate\n",
      "accretropin\n",
      "accrufer\n",
      "accuneb\n",
      "accupril\n",
      "accurbron\n",
      "accuretic\n",
      "accutane\n",
      "acebutolol hydrochloride\n"
     ]
    }
   ],
   "source": [
    "# read in all the potential drug names and active ingredients\n",
    "# dataset from: https://www.fda.gov/drugs/drug-approvals-and-databases/drugsfda-data-files\n",
    "drug_names = set()\n",
    "with open('fda_approved/Products.txt') as infile:\n",
    "    header = infile.readline().strip().split('\\t')\n",
    "    name_ind = header.index('DrugName')\n",
    "    actv_ing_ind = header.index('ActiveIngredient')\n",
    "    # now go through and collect them all\n",
    "    # TODO handle ; separated and parenthetical lists\n",
    "    # TODO e.g. \"drug;drug;drug;\" and \"drug (alias;alias;alias)\"\n",
    "    for l in infile:\n",
    "        parts = l.strip().lower().split('\\t')\n",
    "        drug_names.add(parts[name_ind])\n",
    "        drug_names.add(parts[actv_ing_ind])\n",
    "# sort them into a list\n",
    "drug_names = sorted(drug_names)\n",
    "print(len(drug_names))\n",
    "print('\\n'.join(drug_names[:50]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "actonel with calcium (copackaged)\n",
      "aminosyn 10% (ph6)\n",
      "aminosyn 7% (ph6)\n",
      "aminosyn 8.5% (ph6)\n",
      "antara (micronized)\n",
      "atropine (autoinjector)\n",
      "clobetasol propionate (emollient)\n",
      "cytoxan (lyophilized)\n",
      "ddavp (needs no refrigeration)\n",
      "desmopressin acetate (needs no refrigeration)\n",
      "dextrose 5%, sodium chloride 0.2% and potassium chloride 15meq (k)\n",
      "dextrose 5%, sodium chloride 0.2% and potassium chloride 20meq (k)\n",
      "dextrose 5%, sodium chloride 0.2% and potassium chloride 5meq (k)\n",
      "dextrose 5%, sodium chloride 0.45% and potassium chloride 20meq (k) in plastic container\n",
      "dicyclomine hydrochloride (preservative free)\n",
      "doxil (liposomal)\n",
      "doxorubicin hydrochloride (liposomal)\n",
      "enoxaparin sodium (preservative free)\n",
      "epinephrine (autoinjector)\n",
      "epinephrine (copackaged)\n",
      "evzio (autoinjector)\n",
      "excedrin (migraine)\n",
      "famotidine preservative free (pharmacy bulk)\n",
      "fenofibrate (micronized)\n",
      "ferric hexacyanoferrate(ii)\n",
      "glyburide (micronized)\n",
      "hc (hydrocortisone)\n",
      "hydroserpine plus (r-h-h)\n",
      "infuvite pediatric (pharmacy bulk package)\n",
      "kisqali femara co-pack (copackaged)\n",
      "lanadelumab (shp643)\n",
      "liotrix (t4;t3)\n",
      "lovenox (preservative free)\n",
      "m.v.i. adult (pharmacy bulk package)\n",
      "makena (autoinjector)\n",
      "menotropins (fsh;lh)\n",
      "meretek ubt kit (w/ pranactin)\n",
      "minoxidil (for men)\n",
      "minoxidil (for women)\n",
      "minoxidil extra strength (for men)\n",
      "monistat 3 combination pack (prefilled)\n",
      "nicorette (mint)\n",
      "nitrofurantoin (monohydrate/macrocrystals)\n",
      "norethindrone and ethinyl estradiol (10/11)\n",
      "norethindrone and ethinyl estradiol (7/14)\n",
      "octreotide acetate (preservative free)\n",
      "omnipen (ampicillin)\n",
      "oxycodone and aspirin (half-strength)\n",
      "pancrelipase (amylase;lipase;protease)\n",
      "pravigard pac (copackaged)\n",
      "premphase (premarin;cycrin 14/14)\n",
      "prempro (premarin;cycrin)\n",
      "prevacid naprapac 250 (copackaged)\n",
      "prevacid naprapac 375 (copackaged)\n",
      "prevacid naprapac 500 (copackaged)\n",
      "radiogardase (prussian blue)\n",
      "rituximab; hyaluronidase (human recombinant)\n",
      "rogaine (for men)\n",
      "rogaine (for women)\n",
      "rogaine extra strength (for men)\n",
      "stimate (needs no refrigeration)\n",
      "symdeko (copackaged)\n",
      "tobramycin sulfate (pharmacy bulk)\n",
      "tricor (micronized)\n",
      "trikafta (copackaged)\n",
      "triple sulfa (sulfabenzamide;sulfacetamide;sulfathiazole)\n",
      "trisulfapyrimidines (sulfadiazine;sulfamerazine;sulfamethazine)\n",
      "ultravist (pharmacy bulk)\n",
      "viekira pak (copackaged)\n",
      "vyleesi (autoinjector)\n",
      "xyosted (autoinjector)\n"
     ]
    }
   ],
   "source": [
    "# names with aliases and no parentheticals are easy\n",
    "# e.g abacavir;lamivudine (just split those)\n",
    "# names with parentheticals are harder - what do they look like\n",
    "for dn in drug_names:\n",
    "    if '(' in dn:\n",
    "        print(dn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "premarin: True\n",
      "cycrin: True\n",
      "hydrocortisone: True\n",
      "ampicillin: False\n",
      "sulfabenzamide: False\n",
      "sulfacetamide: False\n",
      "sulfathiazole: False\n",
      "sulfadiazine: True\n",
      "sulfamerazine: False\n",
      "sulfamethazine: False\n"
     ]
    }
   ],
   "source": [
    "# thankfully there aren't many, and most aren't drug or alias names\n",
    "# maybe the relevant drugs are already present and we can just drop ()s\n",
    "drug_name_set = set(drug_names)\n",
    "names_to_check = ['premarin', 'cycrin', 'hydrocortisone', 'ampicillin',\n",
    "                 'sulfabenzamide', 'sulfacetamide', 'sulfathiazole',\n",
    "                 'sulfadiazine', 'sulfamerazine', 'sulfamethazine']\n",
    "for ntc in names_to_check:\n",
    "    print('{0}: {1}'.format(ntc, ntc in drug_name_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8-hour bayer\n",
      "8-mop\n",
      "a-hydrocort\n",
      "a-methapred\n",
      "a-poxide\n",
      "a.p.l.\n",
      "a/t/s\n",
      "abacavir\n",
      "abacavir and lamivudine\n",
      "abacavir sulfate\n",
      "abacavir sulfate and lamivudine\n",
      "abacavir sulfate, lamivudine and zidovudine\n",
      "abaloparatide\n",
      "abarelix\n",
      "abatacept\n",
      "abciximab\n",
      "abelcet\n",
      "abemaciclib\n",
      "abilify\n",
      "abilify maintena kit\n",
      "abilify mycite kit\n",
      "abiraterone acetate\n",
      "abitrexate\n",
      "ablavar\n",
      "ablysinol\n",
      "abobotulinumtoxina\n",
      "abraxane\n",
      "abreva\n",
      "abrilada\n",
      "absorica\n",
      "absorica ld\n",
      "abstral\n",
      "acalabrutinib\n",
      "acamprosate calcium\n",
      "acanya\n",
      "acarbose\n",
      "accolate\n",
      "accretropin\n",
      "accrufer\n",
      "accuneb\n",
      "accupril\n",
      "accurbron\n",
      "accuretic\n",
      "accutane\n",
      "acebutolol hydrochloride\n",
      "aceon\n",
      "acephen\n",
      "acetadote\n",
      "acetaminophen\n",
      "acetaminophen and codeine phosphate\n"
     ]
    }
   ],
   "source": [
    "# okay, to process these into individual drugs, let's do this\n",
    "# 1. drop all text after the first occurrence of a '('\n",
    "# 2. split remaining text by ';'\n",
    "# 3. manually add our list of drugs from ()s [names_to_check above]\n",
    "final_drug_names = set()\n",
    "for dn in drug_names:\n",
    "    dn = dn.strip()\n",
    "    # drop all text following a paren\n",
    "    pindex = dn.find('(')\n",
    "    if pindex >= 0:\n",
    "        dn = dn[:pindex]\n",
    "    # and split by ';' for multiple names\n",
    "    dn = dn.split(';')\n",
    "    for n in dn:\n",
    "        final_drug_names.add(n.strip())\n",
    "# and add all of our manually gathered names\n",
    "for n in names_to_check:\n",
    "    final_drug_names.add(n)\n",
    "# how'd we do?\n",
    "final_drug_names = sorted(final_drug_names)\n",
    "print('\\n'.join(final_drug_names[:50]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fine, let's dump those for use in our word vector stuff\n",
    "with open('fda_approved/fda_approved.processed.names', 'w') as outfile:\n",
    "    outfile.write('\\n'.join(final_drug_names))\n",
    "    outfile.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('penbutolol', 0.8522176146507263),\n",
       " ('hydrochloride-timolol', 0.8416755199432373),\n",
       " ('hydrochlorothizide', 0.8413857221603394),\n",
       " ('oxprenolol', 0.8394852876663208),\n",
       " ('atenolol', 0.8310449719429016),\n",
       " ('bufuralol-hydrochloride', 0.82554030418396),\n",
       " ('hydroxypropranolol', 0.8228106498718262),\n",
       " ('dimethylpropranolol', 0.8134440183639526),\n",
       " ('hydrochlorid', 0.8131069540977478),\n",
       " ('tiapride-hydrochloride', 0.8115503787994385)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's also just play around with multi-token drug names\n",
    "# what are the most similar vectors to the multiple tokens?\n",
    "wv.most_similar(positive=['acebutolol', 'hydrochloride'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('acetaminophen-containing', 0.8329548835754395),\n",
       " ('acetaminophens', 0.8154647350311279),\n",
       " ('p-acetaminophen', 0.812239408493042),\n",
       " ('acetaminophen-sensitive', 0.8013252019882202),\n",
       " ('dextropropoxyphene-containing', 0.8009302616119385),\n",
       " ('non-acetaminophen', 0.7994862198829651),\n",
       " ('post-acetaminophen', 0.7923027276992798),\n",
       " ('paracetamol-containing', 0.791881799697876),\n",
       " ('acetaminophen-sulfate', 0.7916483879089355),\n",
       " ('paracetamol', 0.7904905080795288)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wv.most_similar(positive=['acetaminophen', 'and', 'codeine', 'phosphate'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.3719134\n",
      "3.927033\n"
     ]
    }
   ],
   "source": [
    "# how does this work though? just a normalized average?\n",
    "ace_v = wv['acebutolol']\n",
    "ace_norm = np.linalg.norm(ace_v)\n",
    "ace_v_unit = ace_v / ace_norm\n",
    "hcl_v = wv['hydrochloride']\n",
    "hcl_norm = np.linalg.norm(hcl_v)\n",
    "hcl_v_unit = hcl_v / hcl_norm\n",
    "print(ace_norm)\n",
    "print(hcl_norm)\n",
    "unit_then_avg = (ace_v_unit + hcl_v_unit) / 2.0\n",
    "unit_then_avg = unit_then_avg / np.linalg.norm(unit_then_avg)\n",
    "avg_then_unit = (ace_v + hcl_v) / 2.0\n",
    "avg_then_unit = avg_then_unit / np.linalg.norm(avg_then_unit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('hydrochloride', 0.8997487425804138),\n",
       " ('acebutolol', 0.899748682975769),\n",
       " ('penbutolol', 0.8522176146507263),\n",
       " ('hydrochloride-timolol', 0.8416755199432373),\n",
       " ('hydrochlorothizide', 0.8413857221603394),\n",
       " ('oxprenolol', 0.8394852876663208),\n",
       " ('atenolol', 0.8310450315475464),\n",
       " ('bufuralol-hydrochloride', 0.82554030418396),\n",
       " ('hydroxypropranolol', 0.8228106498718262),\n",
       " ('dimethylpropranolol', 0.8134440183639526)]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# do we scale them to unit vectors before averaging?\n",
    "wv.most_similar(np.array([unit_then_avg]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('acebutolol', 0.9107880592346191),\n",
       " ('hydrochloride', 0.8881013989448547),\n",
       " ('penbutolol', 0.859885573387146),\n",
       " ('oxprenolol', 0.8477539420127869),\n",
       " ('hydrochlorothizide', 0.8413622975349426),\n",
       " ('hydrochloride-timolol', 0.8393110036849976),\n",
       " ('atenolol', 0.8377487063407898),\n",
       " ('hydroxypropranolol', 0.825943112373352),\n",
       " ('bufuralol-hydrochloride', 0.8228414058685303),\n",
       " ('metoprolol', 0.8174272179603577)]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# or scale their average to a unit vector afterward?\n",
    "wv.most_similar(np.array([avg_then_unit]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# it looks like the most_similar function does the former\n",
    "# scale the vectors to unit, average, and then re-unit scale\n",
    "# makes sense really so that one vector doesn't dominate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}