Merge branch 'master' into type-weight

.travis.yml

@@ -0,0 +1,17 @@
+language: python
+python:
+  - "2.7"
+  - "3.4"
+  - "3.6"
+# command to install dependencies
+install:
+  - pip install -Ur test_requirements.txt
+  - pip install .
+# command to run tests
+script:
+  - pytest neleval/
+  - ./shell_tests.sh
+  - pip install -r doc/requirements.txt sphinx_rtd_theme && cd doc && SPHINXOPTS=-W make html ; cd ..
+  - pyflakes neleval/
+after_success:
+  - coveralls

README.rst

@@ -0,0 +1,89 @@
+Entity linking evaluation
+=========================
+
+Python evaluation scripts for `TAC <http://www.nist.gov/tac/>`__ entity
+linking and related wikification, named entity disambiguation, and
+within- and cross-document coreference tasks.
+
+|version| |licence| |py-versions|
+
+|issues| |build| |docs| |coverage|
+
+It aims for **fast** and **flexible** coreference resolution and
+**sophisticated** named entity recognition evaluation, such as partial scores
+for partial overlap between gold and system mentions. CEAF, in particular, is
+much faster to calculate here than in the `CoNLL-11/12 scorer
+<https://github.com/conll/reference-coreference-scorers>`__. It boasts features
+such as configurable metrics; accounting for or ignoring cross-document
+coreference (see the ``evaluate --by-doc`` flag); plotting to compare
+evaluation by system, measure and corpus subset; and bootstrap-based confidence
+interval calculation for document-wise evaluation metrics.
+
+Requires that ``python`` (2.7, with Py3k support experimental/partial)
+be installed on your system with ``numpy`` (and preferably ``scipy`` for
+fast CEAF calculation) and ``joblib``. ``matplotlib`` is required for
+the ``plot-systems`` command.
+
+See a list of commands with:
+
+.. code:: bash
+
+    ./nel --help
+
+Or install onto your Python path (e.g. with
+``pip install git+https://github.com/wikilinks/neleval``) then
+
+.. code:: bash
+
+    python -m neleval --help
+
+TAC-KBP 2014 EDL quickstart
+===========================
+
+.. code:: bash
+
+    ./scripts/run_tac14_evaluation.sh \
+        /path/to/gold.xml \              # TAC14 gold standard queries/mentions
+        /path/to/gold.tab \              # TAC14 gold standard link and nil annotations
+        /system/output/directory \       # directory containing (only) TAC14 system output files
+        /script/output/directory \       # directory to which results are written
+        number_of_jobs                   # number of jobs for parallel mode
+
+Each file in in the system output directory is scored against gold.tab.
+
+Similar facility is available for TAC-KBP'15 EDL.
+
+More details
+============
+
+See `the documentation <https://neleval.readthedocs.io>`__ for more
+details.
+
+
+.. |py-versions| image:: https://img.shields.io/pypi/pyversions/neleval.svg
+    :alt: Python versions supported
+
+.. |version| image:: https://badge.fury.io/py/neleval.svg
+    :alt: Latest version on PyPi
+    :target: https://badge.fury.io/py/neleval
+
+.. |build| image:: https://travis-ci.org/wikilinks/neleval.svg?branch=master
+    :alt: Travis CI build status
+    :scale: 100%
+    :target: https://travis-ci.org/wikilinks/neleval
+
+.. |issues| image:: https://img.shields.io/github/issues/wikilinks/neleval.svg
+    :alt: Issue tracker
+    :target: https://github.com/wikilinks/neleval
+
+.. |coverage| image:: https://coveralls.io/repos/github/wikilinks/neleval/badge.svg
+    :alt: Test coverage
+    :target: https://coveralls.io/github/wikilinks/neleval
+
+.. |docs| image:: https://readthedocs.org/projects/neleval/badge/?version=latest
+     :alt: Documentation Status
+     :scale: 100%
+     :target: https://neleval.readthedocs.io/en/latest/?badge=latest
+
+.. |licence| image:: https://img.shields.io/badge/Licence-BSD-blue.svg
+     :target: https://opensource.org/licenses/BSD-3-Clause

doc/Makefile

@@ -0,0 +1,184 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = _build
+
+# User-friendly check for sphinx-build
+ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
+$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
+endif
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  xml        to make Docutils-native XML files"
+	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+	-rm -rf $(BUILDDIR)/*
+	-rm -rf auto_examples/
+	-rm -rf generated/*
+	-rm -rf modules/generated/*
+
+html:
+	# These two lines make the build a bit more lengthy, and the
+	# the embedding of images more robust
+	rm -rf $(BUILDDIR)/html/_images
+	#rm -rf _build/doctrees/
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/project-template.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/project-template.qhc"
+
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/project-template"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/project-template"
+	@echo "# devhelp"
+
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+latexpdfja:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through platex and dvipdfmx..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
+
+xml:
+	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
+	@echo
+	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
+
+pseudoxml:
+	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
+	@echo
+	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."