Revert "Remove Span._recalculate_indices"

This reverts commit 727370c633.

spacy/about.py

@@ -1,6 +1,6 @@
 # fmt: off
 __title__ = "spacy-nightly"
-__version__ = "3.0.0a39"
+__version__ = "3.0.0a39.dev0"
 __download_url__ = "https://github.com/explosion/spacy-models/releases/download"
 __compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"
 __projects__ = "https://github.com/explosion/projects"

spacy/pipeline/_parser_internals/nonproj.pyx

@@ -4,6 +4,8 @@ for doing pseudo-projective parsing implementation uses the HEAD decoration
 scheme.
 """
 from copy import copy
+from pathlib import Path
+import json
 
 from ...tokens.doc cimport Doc, set_children_from_heads
 
@@ -108,7 +110,23 @@ def projectivize(heads, labels):
     return proj_heads, deco_labels
 
 
+LOG_DIR = Path("/tmp/nonproj_log")
+if not LOG_DIR.exists():
+    LOG_DIR.mkdir(parents=True)
+file_num = 0
 cpdef deprojectivize(Doc doc):
+    global file_num
+    # Log the parse
+    heads = []
+    labels = []
+    for i in range(doc.length):
+        heads.append(doc.c[i].head)
+        labels.append(doc.vocab.strings[doc.c[i].dep])
+    texts = [w.text for w in doc]
+    indices = list(range(len(doc)))
+    with (LOG_DIR / f"{file_num}.json").open("w") as file_:
+        file_.write(json.dumps(list(zip(indices, texts, heads, labels)), indent=2))
+    file_num += 1
     # Reattach arcs with decorated labels (following HEAD scheme). For each
     # decorated arc X||Y, search top-down, left-to-right, breadth-first until
     # hitting a Y then make this the new head.
@@ -165,7 +183,12 @@ def _find_new_head(token, headlabel):
     # returns the id of the first descendant with the given label
     # if there is none, return the current head (no change)
     queue = [token.head]
+    n_iter = 0
+    headlabel = token.vocab.strings.as_int(headlabel)
     while queue:
+        n_iter += 1
+        if n_iter >= len(token.doc):
+            raise ValueError("Infinite loop?")
         next_queue = []
         for qtoken in queue:
             for child in qtoken.children:
@@ -173,7 +196,7 @@ def _find_new_head(token, headlabel):
                     continue
                 if child == token:
                     continue
-                if child.dep_ == headlabel:
+                if child.dep == headlabel:
                     return child
                 next_queue.append(child)
         queue = next_queue

spacy/tests/doc/test_doc_api.py

@@ -608,11 +608,14 @@ def test_doc_init_iob():
         doc = Doc(Vocab(), words=words, ents=ents)
 
 
-def test_doc_set_ents_invalid_spans(en_tokenizer):
+@pytest.mark.xfail
+def test_doc_set_ents_spans(en_tokenizer):
     doc = en_tokenizer("Some text about Colombia and the Czech Republic")
     spans = [Span(doc, 3, 4, label="GPE"), Span(doc, 6, 8, label="GPE")]
     with doc.retokenize() as retokenizer:
         for span in spans:
             retokenizer.merge(span)
-    with pytest.raises(IndexError):
-        doc.ents = spans
+    # If this line is uncommented, it works:
+    # print(spans)
+    doc.ents = spans
+    assert [ent.text for ent in doc.ents] == ["Colombia", "Czech Republic"]

spacy/tests/doc/test_retokenize_merge.py

@@ -336,7 +336,6 @@ def test_doc_retokenize_spans_sentence_update_after_merge(en_tokenizer):
         attrs = {"lemma": "none", "ent_type": "none"}
         retokenizer.merge(doc[0:2], attrs=attrs)
         retokenizer.merge(doc[-2:], attrs=attrs)
-    sent1, sent2 = list(doc.sents)
     assert len(sent1) == init_len - 1
     assert len(sent2) == init_len2 - 1
 

spacy/tokens/span.pxd

@@ -16,4 +16,5 @@ cdef class Span:
     cdef public _vector
     cdef public _vector_norm
 
+    cpdef int _recalculate_indices(self) except -1
     cpdef np.ndarray to_array(self, object features)

spacy/tokens/span.pyx

@@ -150,6 +150,7 @@ cdef class Span:
 
         DOCS: https://nightly.spacy.io/api/span#len
         """
+        self._recalculate_indices()
         if self.end < self.start:
             return 0
         return self.end - self.start
@@ -166,6 +167,7 @@ cdef class Span:
 
         DOCS: https://nightly.spacy.io/api/span#getitem
         """
+        self._recalculate_indices()
         if isinstance(i, slice):
             start, end = normalize_slice(len(self), i.start, i.stop, i.step)
             return Span(self.doc, start + self.start, end + self.start)
@@ -186,6 +188,7 @@ cdef class Span:
 
         DOCS: https://nightly.spacy.io/api/span#iter
         """
+        self._recalculate_indices()
         for i in range(self.start, self.end):
             yield self.doc[i]
 
@@ -336,6 +339,19 @@ cdef class Span:
                 output[i-self.start, j] = get_token_attr(&self.doc.c[i], feature)
         return output
 
+    cpdef int _recalculate_indices(self) except -1:
+        if self.end > self.doc.length \
+        or self.doc.c[self.start].idx != self.start_char \
+        or (self.doc.c[self.end-1].idx + self.doc.c[self.end-1].lex.length) != self.end_char:
+            start = token_by_start(self.doc.c, self.doc.length, self.start_char)
+            if self.start == -1:
+                raise IndexError(Errors.E036.format(start=self.start_char))
+            end = token_by_end(self.doc.c, self.doc.length, self.end_char)
+            if end == -1:
+                raise IndexError(Errors.E037.format(end=self.end_char))
+            self.start = start
+            self.end = end + 1
+
     @property
     def vocab(self):
         """RETURNS (Vocab): The Span's Doc's vocab."""
@@ -504,6 +520,7 @@ cdef class Span:
 
         DOCS: https://nightly.spacy.io/api/span#root
         """
+        self._recalculate_indices()
         if "root" in self.doc.user_span_hooks:
             return self.doc.user_span_hooks["root"](self)
         # This should probably be called 'head', and the other one called

spacy/training/loop.py

@@ -7,6 +7,7 @@ from wasabi import Printer
 import random
 import sys
 import shutil
+import itertools
 
 from .example import Example
 from ..schemas import ConfigSchemaTraining