displaCy

{ "cells": [ { "cell_type": "markdown", "id": "3e87b43e-fa2e-408d-b28e-fb2c8fbf5abf", "metadata": {}, "source": [ "# 07 - Named Entity Recognition" ] }, { "cell_type": "markdown", "id": "5d073a4b-122e-47ae-9608-9ac34c02e831", "metadata": {}, "source": [ "NER seeks to locate and classify named entity mentions in undestructured text into pre-defined categories such as person names, organizations, locations, medical codes, time expressions, quantities, monetary values, percentages, etc." ] }, { "cell_type": "code", "execution_count": 1, "id": "cf969f54-4ef1-4551-a28b-6cd0ee53e058", "metadata": {}, "outputs": [], "source": [ "import spacy" ] }, { "cell_type": "code", "execution_count": 2, "id": "ad7893c6-a777-48d0-ab52-8d3f441201e2", "metadata": {}, "outputs": [], "source": [ "nlp = spacy.load('en_core_web_sm')" ] }, { "cell_type": "code", "execution_count": 3, "id": "0410faa5-3618-4355-8e5e-e4574d519f4c", "metadata": {}, "outputs": [], "source": [ "def show_ents(doc):\n", " if (doc.ents):\n", " for ent in doc.ents:\n", " print(f'{ent.text} - {ent.label_} - {spacy.explain(ent.label_)}')\n", " else:\n", " print('No entities found.')" ] }, { "cell_type": "code", "execution_count": 4, "id": "c9a4a9aa-00dd-4132-bc53-c4049c446004", "metadata": {}, "outputs": [], "source": [ "doc = nlp(u'Hi, how are you?')" ] }, { "cell_type": "code", "execution_count": 5, "id": "e9efe010-f6ee-4176-bc86-31aa76b92bf4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "No entities found.\n" ] } ], "source": [ "show_ents(doc)" ] }, { "cell_type": "code", "execution_count": 6, "id": "423edb0d-96db-4514-b2e8-a307ebcb4efc", "metadata": {}, "outputs": [], "source": [ "doc = nlp(u'May i go to Washington, DC next May to see Washington Monument?')" ] }, { "cell_type": "code", "execution_count": 7, "id": "a688baad-2f6f-4b28-9ce1-fcfed77993c2", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Washington, DC - GPE - Countries, cities, states\n", "next May - DATE - Absolute or relative dates or periods\n", "Washington Monument - PERSON - People, including fictional\n" ] } ], "source": [ "show_ents(doc)" ] }, { "cell_type": "code", "execution_count": 8, "id": "33f59ef0-cc44-4f69-92a8-db5a9d25b206", "metadata": {}, "outputs": [], "source": [ "doc = nlp(u'Can i please have 500 dollars of Microsoft stock?')" ] }, { "cell_type": "code", "execution_count": 9, "id": "14985eaa-037d-419d-aea0-6f2a627de416", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "500 dollars - MONEY - Monetary values, including unit\n", "Microsoft - ORG - Companies, agencies, institutions, etc.\n" ] } ], "source": [ "show_ents(doc)" ] }, { "cell_type": "code", "execution_count": 10, "id": "a530aee0-2c53-4d85-a327-c507d140b92d", "metadata": {}, "outputs": [], "source": [ "doc = nlp(u'Tesla to build a U.K. factory for $6 million.')" ] }, { "cell_type": "markdown", "id": "48242b63-aa30-4a41-aca2-60d02eb59015", "metadata": {}, "source": [ "## Adding a new named entity to Spacy" ] }, { "cell_type": "code", "execution_count": 11, "id": "81ed966f-8834-47df-a74c-cf13874ec054", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "U.K. - GPE - Countries, cities, states\n", "$6 million - MONEY - Monetary values, including unit\n" ] } ], "source": [ "show_ents(doc)" ] }, { "cell_type": "code", "execution_count": 12, "id": "797d425b-3db2-43dc-b9cd-557b238ff595", "metadata": {}, "outputs": [], "source": [ "from spacy.tokens import Span" ] }, { "cell_type": "code", "execution_count": 13, "id": "423ca277-eb09-4ebf-9ea7-cb1c18adb8b0", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "383" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "org = doc.vocab.strings[u\"ORG\"]\n", "org" ] }, { "cell_type": "code", "execution_count": 14, "id": "f9c0e200-dc7f-4f56-b8a6-c4a98e13d5be", "metadata": {}, "outputs": [], "source": [ "tesla_entity = Span(\n", " # doc wich includes the new named entity\n", " doc,\n", " # starting index of the span\n", " 0,\n", " # ending index of the span\n", " 1,\n", " # named entity category\n", " label=org\n", ")" ] }, { "cell_type": "code", "execution_count": 15, "id": "2ab71f8e-7fb1-4ecb-8f61-12cfd6e57cd3", "metadata": {}, "outputs": [], "source": [ "doc.ents = list(doc.ents) + [tesla_entity]" ] }, { "cell_type": "code", "execution_count": 16, "id": "8581fa90-2ef8-408d-853f-75e53ed01659", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Tesla - ORG - Companies, agencies, institutions, etc.\n", "U.K. - GPE - Countries, cities, states\n", "$6 million - MONEY - Monetary values, including unit\n" ] } ], "source": [ "show_ents(doc)" ] }, { "cell_type": "markdown", "id": "4fd4102d-46a5-420f-a0e0-438e324b753f", "metadata": {}, "source": [ "## Adding multiple named entities" ] }, { "cell_type": "code", "execution_count": 17, "id": "8e3af36d-f77b-4bd9-8b74-5b45905d16c0", "metadata": {}, "outputs": [], "source": [ "doc = nlp(u'Our company created a new vacuum cleaner.'\n", " u'This new vacuum-cleaner is the best in the show.')" ] }, { "cell_type": "code", "execution_count": 18, "id": "a0e38e65-53cd-490a-a931-566fba813f3e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "No entities found.\n" ] } ], "source": [ "show_ents(doc)" ] }, { "cell_type": "code", "execution_count": 19, "id": "f340cdcc-8a1e-4acd-92e0-3e5ca366b9da", "metadata": {}, "outputs": [], "source": [ "from spacy.matcher import PhraseMatcher" ] }, { "cell_type": "code", "execution_count": 20, "id": "4a5f13b5-bad0-46a7-a057-22494e065c81", "metadata": {}, "outputs": [], "source": [ "matcher = PhraseMatcher(nlp.vocab)" ] }, { "cell_type": "code", "execution_count": 21, "id": "527bf1f7-1b11-4e36-be2f-7ef0bc0c28c7", "metadata": {}, "outputs": [], "source": [ "phrase_list = ['vacuum-cleaner', 'vacuum cleaner']" ] }, { "cell_type": "code", "execution_count": 22, "id": "68a6008a-a054-41d8-a5c8-4fa6118996f5", "metadata": {}, "outputs": [], "source": [ "phrase_patterns = [nlp(text) for text in phrase_list]" ] }, { "cell_type": "code", "execution_count": 23, "id": "23c6f7e6-8cb6-4de5-a672-c529c0a78333", "metadata": {}, "outputs": [], "source": [ "matcher.add('newproduct',None,*phrase_patterns)" ] }, { "cell_type": "code", "execution_count": 24, "id": "b9a72a30-c06b-4ff7-8b30-45c4474fa305", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[(2689272359382549672, 5, 7), (2689272359382549672, 10, 13)]" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "matches = matcher(doc)\n", "matches" ] }, { "cell_type": "markdown", "id": "05535858-92c1-4785-a0c7-d7b8c4660a9e", "metadata": {}, "source": [ "Create a span for each match:" ] }, { "cell_type": "code", "execution_count": 25, "id": "070afc18-d1e3-490d-80f3-1cff9806ec08", "metadata": {}, "outputs": [], "source": [ "PROD = doc.vocab.strings[u\"PRODUCT\"]" ] }, { "cell_type": "code", "execution_count": 26, "id": "b62a0b64-8f89-4a5a-96bc-e5a72f1306d3", "metadata": {}, "outputs": [], "source": [ "new_ents = [Span(doc,match[1],match[2],label=PROD) for match in matches]" ] }, { "cell_type": "code", "execution_count": 27, "id": "854c96e3-fe66-4b93-b05d-bf248184bad9", "metadata": {}, "outputs": [], "source": [ "doc.ents = list(doc.ents) + new_ents" ] }, { "cell_type": "code", "execution_count": 28, "id": "55cfe689-af33-46d9-bd69-cb6fee7c3ec3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "vacuum cleaner - PRODUCT - Objects, vehicles, foods, etc. (not services)\n", "vacuum-cleaner - PRODUCT - Objects, vehicles, foods, etc. (not services)\n" ] } ], "source": [ "show_ents(doc)" ] }, { "cell_type": "code", "execution_count": 29, "id": "18560092-d32b-4bc1-8094-23eed6ea3573", "metadata": {}, "outputs": [], "source": [ "doc = nlp(u\"Originally i paid $29.95 for this car toy, but now it is marked down by 10 dollars\")" ] }, { "cell_type": "markdown", "id": "be7cfe40-1637-40d3-81a4-c59bbf2653fe", "metadata": {}, "source": [ "Retrieving the count of the occurrences:" ] }, { "cell_type": "code", "execution_count": 30, "id": "ef22785e-5694-4a6c-a282-6cd5a1e43bb1", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "2" ] }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len([ent for ent in doc.ents if ent.label_ == \"MONEY\"])" ] }, { "cell_type": "markdown", "id": "c7f0d0f3-56fe-47ac-9c02-b07d9e42d145", "metadata": {}, "source": [ "## Visualizing NER" ] }, { "cell_type": "code", "execution_count": 31, "id": "286e327d-4ac9-458e-bed7-6d01d2a42391", "metadata": {}, "outputs": [], "source": [ "from spacy import displacy" ] }, { "cell_type": "code", "execution_count": 32, "id": "df2a9d18-e3f1-44db-82ab-be649fde59ea", "metadata": {}, "outputs": [], "source": [ "doc = nlp(\n", " u'Over the last quarter Apple sold nearly 20 thousand iPods for a profit of $6 million.'\n", " u'By contrast, Sony only sold 8 thousand Walkman music players.'\n", ")" ] }, { "cell_type": "code", "execution_count": 33, "id": "d84439ab-d27f-4b34-a2df-d024271b0cf3", "metadata": {}, "outputs": [ { "data": { "text/html": [ "

Over \n", "\n", " the last quarter\n", " DATE\n", "\n", " \n", "\n", " Apple\n", " ORG\n", "\n", " sold \n", "\n", " nearly 20 thousand\n", " CARDINAL\n", "\n", " \n", "\n", " iPods\n", " PRODUCT\n", "\n", " for a profit of \n", "\n", " $6 million\n", " MONEY\n", "\n", ".By contrast, \n", "\n", " Sony\n", " ORG\n", "\n", " only sold \n", "\n", " 8 thousand\n", " CARDINAL\n", "\n", " Walkman music players.

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "displacy.render(doc, style=\"ent\", jupyter=True)" ] }, { "cell_type": "code", "execution_count": 34, "id": "91ec0481-f729-4f1e-9ec6-35b586aa2cb4", "metadata": {}, "outputs": [ { "data": { "text/html": [ "

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "

By contrast, \n", "\n", " Sony\n", " ORG\n", "\n", " only sold \n", "\n", " 8 thousand\n", " CARDINAL\n", "\n", " Walkman music players.

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "for sent in doc.sents:\n", " displacy.render(nlp(sent.text), style='ent', jupyter=True)" ] }, { "cell_type": "markdown", "id": "59c461e5-c21e-439d-bf82-294a7d2cfe5a", "metadata": {}, "source": [ "Highlighting only certain entities:" ] }, { "cell_type": "code", "execution_count": 35, "id": "00c855e4-4f38-4be4-8f81-2dc0c596f596", "metadata": {}, "outputs": [], "source": [ "colors = {'ORG': 'linear-gradient(45deg, orange, red)'}\n", "options = {'ents': ['PRODUCT', 'ORG'], 'colors': colors }" ] }, { "cell_type": "code", "execution_count": 36, "id": "8dd8b833-cc4e-47da-9a5a-179e3a446f2a", "metadata": {}, "outputs": [ { "data": { "text/html": [ "

Over the last quarter \n", "\n", " Apple\n", " ORG\n", "\n", " sold nearly 20 thousand \n", "\n", " iPods\n", " PRODUCT\n", "\n", " for a profit of $6 million.By contrast, \n", "\n", " Sony\n", " ORG\n", "\n", " only sold 8 thousand Walkman music players.

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "displacy.render(doc, style='ent', jupyter=True, options=options)" ] }, { "cell_type": "code", "execution_count": 37, "id": "3906eef1-59b6-4ff4-8ce1-a6a20fe25964", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/conda/lib/python3.11/site-packages/spacy/displacy/__init__.py:106: UserWarning: [W011] It looks like you're calling displacy.serve from within a Jupyter notebook or a similar environment. This likely means you're already running a local web server, so there's no need to make displaCy start another one. Instead, you should be able to replace displacy.serve with displacy.render to show the visualization.\n", " warnings.warn(Warnings.W011)\n" ] }, { "data": { "text/html": [ "\n", "\n", " \n", " displaCy\n", " \n", "\n", " \n", "

\n", "

\n", "\n", "\n", "" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Using the 'ent' visualizer\n", "Serving on http://0.0.0.0:5000 ...\n", "\n", "Shutting down server on port 5000.\n" ] } ], "source": [ "displacy.serve(doc, style='ent', options=options)" ] }, { "cell_type": "code", "execution_count": null, "id": "a3f24d31-47fa-438e-9633-991d87523848", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.4" } }, "nbformat": 4, "nbformat_minor": 5 }