Inc version

This reverts commit c6df0eafd0.

spacy/about.py

@@ -1,6 +1,6 @@
 # fmt: off
 __title__ = "spacy-nightly"
-__version__ = "3.0.0rc4"
+__version__ = "3.0.0rc4.dev20"
 __download_url__ = "https://github.com/explosion/spacy-models/releases/download"
 __compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"
 __projects__ = "https://github.com/explosion/projects"

spacy/pipeline/_parser_internals/_state.pxd

@@ -32,6 +32,7 @@ cdef cppclass StateC:
     vector[ArcC] _left_arcs
     vector[ArcC] _right_arcs
     vector[libcpp.bool] _unshiftable
+    vector[int] history
     set[int] _sent_starts
     TokenC _empty_token
     int length
@@ -382,3 +383,4 @@ cdef cppclass StateC:
         this._b_i = src._b_i
         this.offset = src.offset
         this._empty_token = src._empty_token
+        this.history = src.history

spacy/pipeline/_parser_internals/arc_eager.pyx

@@ -757,6 +757,8 @@ cdef class ArcEager(TransitionSystem):
         return list(arcs)
 
     def has_gold(self, Example eg, start=0, end=None):
+        if end is not None and end < 0:
+            end = None
         for word in eg.y[start:end]:
             if word.dep != 0:
                 return True
@@ -842,6 +844,7 @@ cdef class ArcEager(TransitionSystem):
                             state.print_state()
                         )))
                     action.do(state.c, action.label)
+                    state.c.history.push_back(i)
                     break
             else:
                 failed = False

spacy/pipeline/_parser_internals/ner.pyx

@@ -266,6 +266,8 @@ cdef class BiluoPushDown(TransitionSystem):
         return BiluoGold(self, state, example)
 
     def has_gold(self, Example eg, start=0, end=None):
+        if end is not None and end < 0:
+            end = None
         for word in eg.y[start:end]:
             if word.ent_iob != 0:
                 return True

spacy/pipeline/_parser_internals/stateclass.pyx

@@ -20,6 +20,10 @@ cdef class StateClass:
         if self._borrowed != 1:
             del self.c
 
+    @property
+    def history(self):
+        return list(self.c.history)
+
     @property
     def stack(self):
         return [self.S(i) for i in range(self.c.stack_depth())]

spacy/pipeline/_parser_internals/transition_system.pyx

@@ -61,7 +61,18 @@ cdef class TransitionSystem:
             offset += len(doc)
         return states
 
+    def follow_history(self, doc, history):
+        cdef int clas
+        cdef StateClass state = StateClass(doc)
+        for clas in history:
+            action = self.c[clas]
+            action.do(state.c, action.label)
+            state.c.history.push_back(clas)
+        return state
+
     def get_oracle_sequence(self, Example example, _debug=False):
+        if not self.has_gold(example):
+            return []
         states, golds, _ = self.init_gold_batch([example])
         if not states:
             return []
@@ -73,6 +84,8 @@ cdef class TransitionSystem:
             return self.get_oracle_sequence_from_state(state, gold)
 
     def get_oracle_sequence_from_state(self, StateClass state, gold, _debug=None):
+        if state.is_final():
+            return []
         cdef Pool mem = Pool()
         # n_moves should not be zero at this point, but make sure to avoid zero-length mem alloc
         assert self.n_moves > 0
@@ -98,6 +111,7 @@ cdef class TransitionSystem:
                             "S0 head?", str(state.has_head(state.S(0))),
                         )))
                     action.do(state.c, action.label)
+                    state.c.history.push_back(i)
                     break
             else:
                 if _debug:
@@ -125,6 +139,7 @@ cdef class TransitionSystem:
             raise ValueError(Errors.E170.format(name=name))
         action = self.lookup_transition(name)
         action.do(state.c, action.label)
+        state.c.history.push_back(action.clas)
 
     cdef Transition lookup_transition(self, object name) except *:
         raise NotImplementedError

spacy/pipeline/transition_parser.pyx

@@ -203,15 +203,21 @@ cdef class Parser(TrainablePipe):
             )
 
     def greedy_parse(self, docs, drop=0.):
-        cdef vector[StateC*] states
-        cdef StateClass state
         set_dropout_rate(self.model, drop)
-        batch = self.moves.init_batch(docs)
         # This is pretty dirty, but the NER can resize itself in init_batch,
         # if labels are missing. We therefore have to check whether we need to
         # expand our model output.
         self._resize()
         model = self.model.predict(docs)
+        batch = self.moves.init_batch(docs)
+        states = self._predict_states(model, batch)
+        model.clear_memory()
+        del model
+        return states
+
+    def _predict_states(self, model, batch):
+        cdef vector[StateC*] states
+        cdef StateClass state
         weights = get_c_weights(model)
         for state in batch:
             if not state.is_final():
@@ -220,8 +226,6 @@ cdef class Parser(TrainablePipe):
         with nogil:
             self._parseC(&states[0],
                 weights, sizes)
-        model.clear_memory()
-        del model
         return batch
 
     def beam_parse(self, docs, int beam_width, float drop=0., beam_density=0.):
@@ -306,6 +310,7 @@ cdef class Parser(TrainablePipe):
             else:
                 action = self.moves.c[guess]
                 action.do(states[i], action.label)
+                states[i].history.push_back(guess)
         free(is_valid)
 
     def update(self, examples, *, drop=0., sgd=None, losses=None):
@@ -316,7 +321,8 @@ cdef class Parser(TrainablePipe):
         validate_examples(examples, "Parser.update")
         for multitask in self._multitasks:
             multitask.update(examples, drop=drop, sgd=sgd)
-    
+        # We need to take care to act on the whole batch, because we might be
+        # getting vectors via a listener.
         n_examples = len([eg for eg in examples if self.moves.has_gold(eg)])
         if n_examples == 0:
             return losses
@@ -332,6 +338,11 @@ cdef class Parser(TrainablePipe):
                 losses=losses,
                 beam_density=self.cfg["beam_density"]
             )
+        model, backprop_tok2vec = self.model.begin_update([eg.x for eg in examples])
+        final_states = self.moves.init_batch([eg.x for eg in examples])
+        self._predict_states(model, final_states)
+        histories = [list(state.history) for state in final_states]
+        #oracle_histories = [self.moves.get_oracle_sequence(eg) for eg in examples]
         max_moves = self.cfg["update_with_oracle_cut_size"]
         if max_moves >= 1:
             # Chop sequences into lengths of this many words, to make the
@@ -339,13 +350,13 @@ cdef class Parser(TrainablePipe):
             max_moves = int(random.uniform(max_moves // 2, max_moves * 2))
             states, golds, _ = self._init_gold_batch(
                 examples,
+                histories,
                 max_length=max_moves
             )
         else:
             states, golds, _ = self.moves.init_gold_batch(examples)
         if not states:
             return losses
-        model, backprop_tok2vec = self.model.begin_update([eg.x for eg in examples])
  
         all_states = list(states)
         states_golds = list(zip(states, golds))
@@ -369,8 +380,7 @@ cdef class Parser(TrainablePipe):
         backprop_tok2vec(golds)
         if sgd not in (None, False):
             self.finish_update(sgd)
-        docs = [eg.predicted for eg in examples]
-        self.set_annotations(docs, all_states)
+        self.set_annotations([eg.x for eg in examples], final_states)
         # Ugh, this is annoying. If we're working on GPU, we want to free the
         # memory ASAP. It seems that Python doesn't necessarily get around to
         # removing these in time if we don't explicitly delete? It's confusing.
@@ -577,7 +587,7 @@ cdef class Parser(TrainablePipe):
                     raise ValueError(Errors.E149) from None
         return self
 
-    def _init_gold_batch(self, examples, max_length):
+    def _init_gold_batch(self, examples, oracle_histories, max_length):
         """Make a square batch, of length equal to the shortest transition
         sequence or a cap. A long
         doc will get multiple states. Let's say we have a doc of length 2*N,
@@ -588,33 +598,28 @@ cdef class Parser(TrainablePipe):
             StateClass state
             Transition action
         all_states = self.moves.init_batch([eg.predicted for eg in examples])
+        assert len(all_states) == len(examples) == len(oracle_histories)
         states = []
         golds = []
-        to_cut = []
-        for state, eg in zip(all_states, examples):
-            if self.moves.has_gold(eg) and not state.is_final():
-                gold = self.moves.init_gold(state, eg)
-                if len(eg.x) < max_length:
-                    states.append(state)
-                    golds.append(gold)
-                else:
-                    oracle_actions = self.moves.get_oracle_sequence_from_state(
-                        state.copy(), gold)
-                    to_cut.append((eg, state, gold, oracle_actions))
-        if not to_cut:
-            return states, golds, 0
-        cdef int clas
-        for eg, state, gold, oracle_actions in to_cut:
-            for i in range(0, len(oracle_actions), max_length):
+        for state, eg, history in zip(all_states, examples, oracle_histories):
+            if not history:
+                continue
+            gold = self.moves.init_gold(state, eg)
+            if len(history) < max_length:
+                states.append(state)
+                golds.append(gold)
+                continue
+            for i in range(0, len(history), max_length):
+                if state.is_final():
+                    break
                 start_state = state.copy()
-                for clas in oracle_actions[i:i+max_length]:
+                for clas in history[i:i+max_length]:
                     action = self.moves.c[clas]
                     action.do(state.c, action.label)
+                    state.c.history.push_back(clas)
                     if state.is_final():
                         break
                 if self.moves.has_gold(eg, start_state.B(0), state.B(0)):
                     states.append(start_state)
                     golds.append(gold)
-                if state.is_final():
-                    break
         return states, golds, max_length

spacy/tokens/_dict_proxies.py

@@ -33,6 +33,9 @@ class SpanGroups(UserDict):
     def _make_span_group(self, name: str, spans: Iterable["Span"]) -> SpanGroup:
         return SpanGroup(self.doc_ref(), name=name, spans=spans)
 
+    def copy(self) -> "SpanGroups":
+        return SpanGroups(self.doc_ref()).from_bytes(self.to_bytes())
+
     def to_bytes(self) -> bytes:
         # We don't need to serialize this as a dict, because the groups
         # know their names.

spacy/tokens/doc.pyx

@@ -261,11 +261,11 @@ cdef class Doc:
         cdef const LexemeC* lexeme
         for word, has_space in zip(words, spaces):
             if isinstance(word, unicode):
-                lexeme = self.vocab.get(self.mem, word)
+                lexeme = self.vocab.get(self.vocab.mem, word)
             elif isinstance(word, bytes):
                 raise ValueError(Errors.E028.format(value=word))
             else:
-                lexeme = self.vocab.get_by_orth(self.mem, word)
+                lexeme = self.vocab.get_by_orth(self.vocab.mem, word)
             self.push_back(lexeme, has_space)
 
         if heads is not None:
@@ -1180,6 +1180,7 @@ cdef class Doc:
         other.tensor = copy.deepcopy(self.tensor)
         other.cats = copy.deepcopy(self.cats)
         other.user_data = copy.deepcopy(self.user_data)
+        other.spans = self.spans.copy()
         other.sentiment = self.sentiment
         other.has_unknown_spaces = self.has_unknown_spaces
         other.user_hooks = dict(self.user_hooks)
@@ -1334,7 +1335,7 @@ cdef class Doc:
             end = start + attrs[i, 0]
             has_space = attrs[i, 1]
             orth_ = text[start:end]
-            lex = self.vocab.get(self.mem, orth_)
+            lex = self.vocab.get(self.vocab.mem, orth_)
             self.push_back(lex, has_space)
             start = end + has_space
         self.from_array(msg["array_head"][2:], attrs[:, 2:])

spacy/vocab.pyx

@@ -161,8 +161,11 @@ cdef class Vocab:
             return self._new_lexeme(mem, self.strings[orth])
 
     cdef const LexemeC* _new_lexeme(self, Pool mem, unicode string) except NULL:
-        if len(string) < 3 or self.length < 10000:
-            mem = self.mem
+        #if len(string) < 3 or self.length < 10000:
+        #    mem = self.mem
+        # TODO: Experiment with never allowing the Doc to own lexemes, to see
+        # if it solves the Doc.copy() issue.
+        mem = self.mem
         cdef bint is_oov = mem is not self.mem
         lex = <LexemeC*>mem.alloc(1, sizeof(LexemeC))
         lex.orth = self.strings.add(string)