WIP on porting linear parser to v2

setup.py

@@ -29,7 +29,7 @@ MOD_NAMES = [
     'spacy.syntax._state',
     'spacy.syntax._beam_utils',
     'spacy.tokenizer',
-    'spacy.syntax.nn_parser',
+    'spacy.syntax.parser',
     'spacy.syntax.nonproj',
     'spacy.syntax.transition_system',
     'spacy.syntax.arc_eager',

spacy/syntax/_parse_features.pxd

@@ -0,0 +1,259 @@
+from thinc.typedefs cimport atom_t
+
+from .stateclass cimport StateClass
+from ._state cimport StateC
+
+
+cdef int fill_context(atom_t* context, const StateC* state) nogil
+# Context elements
+
+# Ensure each token's attributes are listed: w, p, c, c6, c4. The order
+# is referenced by incrementing the enum...
+
+# Tokens are listed in left-to-right order.
+#cdef size_t* SLOTS = [
+#    S2w, S1w,
+#    S0l0w, S0l2w, S0lw,
+#    S0w,
+#    S0r0w, S0r2w, S0rw,
+#    N0l0w, N0l2w, N0lw,
+#    P2w, P1w,
+#    N0w, N1w, N2w, N3w, 0
+#]
+
+# NB: The order of the enum is _NOT_ arbitrary!!
+cpdef enum:
+    S2w
+    S2W
+    S2p
+    S2c
+    S2c4
+    S2c6
+    S2L
+    S2_prefix
+    S2_suffix
+    S2_shape
+    S2_ne_iob
+    S2_ne_type
+
+    S1w
+    S1W
+    S1p
+    S1c
+    S1c4
+    S1c6
+    S1L
+    S1_prefix
+    S1_suffix
+    S1_shape
+    S1_ne_iob
+    S1_ne_type
+
+    S1rw
+    S1rW
+    S1rp
+    S1rc
+    S1rc4
+    S1rc6
+    S1rL
+    S1r_prefix
+    S1r_suffix
+    S1r_shape
+    S1r_ne_iob
+    S1r_ne_type
+
+    S0lw
+    S0lW
+    S0lp
+    S0lc
+    S0lc4
+    S0lc6
+    S0lL
+    S0l_prefix
+    S0l_suffix
+    S0l_shape
+    S0l_ne_iob
+    S0l_ne_type
+
+    S0l2w
+    S0l2W
+    S0l2p
+    S0l2c
+    S0l2c4
+    S0l2c6
+    S0l2L
+    S0l2_prefix
+    S0l2_suffix
+    S0l2_shape
+    S0l2_ne_iob
+    S0l2_ne_type
+
+    S0w
+    S0W
+    S0p
+    S0c
+    S0c4
+    S0c6
+    S0L
+    S0_prefix
+    S0_suffix
+    S0_shape
+    S0_ne_iob
+    S0_ne_type
+
+    S0r2w
+    S0r2W
+    S0r2p
+    S0r2c
+    S0r2c4
+    S0r2c6
+    S0r2L
+    S0r2_prefix
+    S0r2_suffix
+    S0r2_shape
+    S0r2_ne_iob
+    S0r2_ne_type
+
+    S0rw
+    S0rW
+    S0rp
+    S0rc
+    S0rc4
+    S0rc6
+    S0rL
+    S0r_prefix
+    S0r_suffix
+    S0r_shape
+    S0r_ne_iob
+    S0r_ne_type
+
+    N0l2w
+    N0l2W
+    N0l2p
+    N0l2c
+    N0l2c4
+    N0l2c6
+    N0l2L
+    N0l2_prefix
+    N0l2_suffix
+    N0l2_shape
+    N0l2_ne_iob
+    N0l2_ne_type
+
+    N0lw
+    N0lW
+    N0lp
+    N0lc
+    N0lc4
+    N0lc6
+    N0lL
+    N0l_prefix
+    N0l_suffix
+    N0l_shape
+    N0l_ne_iob
+    N0l_ne_type
+
+    N0w
+    N0W
+    N0p
+    N0c
+    N0c4
+    N0c6
+    N0L
+    N0_prefix
+    N0_suffix
+    N0_shape
+    N0_ne_iob
+    N0_ne_type
+
+    N1w
+    N1W
+    N1p
+    N1c
+    N1c4
+    N1c6
+    N1L
+    N1_prefix
+    N1_suffix
+    N1_shape
+    N1_ne_iob
+    N1_ne_type
+
+    N2w
+    N2W
+    N2p
+    N2c
+    N2c4
+    N2c6
+    N2L
+    N2_prefix
+    N2_suffix
+    N2_shape
+    N2_ne_iob
+    N2_ne_type
+
+    P1w
+    P1W
+    P1p
+    P1c
+    P1c4
+    P1c6
+    P1L
+    P1_prefix
+    P1_suffix
+    P1_shape
+    P1_ne_iob
+    P1_ne_type
+
+    P2w
+    P2W
+    P2p
+    P2c
+    P2c4
+    P2c6
+    P2L
+    P2_prefix
+    P2_suffix
+    P2_shape
+    P2_ne_iob
+    P2_ne_type
+
+    E0w
+    E0W
+    E0p
+    E0c
+    E0c4
+    E0c6
+    E0L
+    E0_prefix
+    E0_suffix
+    E0_shape
+    E0_ne_iob
+    E0_ne_type
+
+    E1w
+    E1W
+    E1p
+    E1c
+    E1c4
+    E1c6
+    E1L
+    E1_prefix
+    E1_suffix
+    E1_shape
+    E1_ne_iob
+    E1_ne_type
+
+    # Misc features at the end
+    dist
+    N0lv
+    S0lv
+    S0rv
+    S1lv
+    S1rv
+
+    S0_has_head
+    S1_has_head
+    S2_has_head
+
+    CONTEXT_SIZE

spacy/syntax/_parse_features.pyx

@@ -0,0 +1,419 @@
+"""
+Fill an array, context, with every _atomic_ value our features reference.
+We then write the _actual features_ as tuples of the atoms. The machinery
+that translates from the tuples to feature-extractors (which pick the values
+out of "context") is in features/extractor.pyx
+
+The atomic feature names are listed in a big enum, so that the feature tuples
+can refer to them.
+"""
+# coding: utf-8
+from __future__ import unicode_literals
+
+from libc.string cimport memset
+from itertools import combinations
+from cymem.cymem cimport Pool
+
+from ..structs cimport TokenC
+from .stateclass cimport StateClass
+from ._state cimport StateC
+
+
+cdef inline void fill_token(atom_t* context, const TokenC* token) nogil:
+    if token is NULL:
+        context[0] = 0
+        context[1] = 0
+        context[2] = 0
+        context[3] = 0
+        context[4] = 0
+        context[5] = 0
+        context[6] = 0
+        context[7] = 0
+        context[8] = 0
+        context[9] = 0
+        context[10] = 0
+        context[11] = 0
+    else:
+        context[0] = token.lex.orth
+        context[1] = token.lemma
+        context[2] = token.tag
+        context[3] = token.lex.cluster
+        # We've read in the string little-endian, so now we can take & (2**n)-1
+        # to get the first n bits of the cluster.
+        # e.g. s = "1110010101"
+        # s = ''.join(reversed(s))
+        # first_4_bits = int(s, 2)
+        # print first_4_bits
+        # 5
+        # print "{0:b}".format(prefix).ljust(4, '0')
+        # 1110
+        # What we're doing here is picking a number where all bits are 1, e.g.
+        # 15 is 1111, 63 is 111111 and doing bitwise AND, so getting all bits in
+        # the source that are set to 1.
+        context[4] = token.lex.cluster & 15
+        context[5] = token.lex.cluster & 63
+        context[6] = token.dep if token.head != 0 else 0
+        context[7] = token.lex.prefix
+        context[8] = token.lex.suffix
+        context[9] = token.lex.shape
+        context[10] = token.ent_iob
+        context[11] = token.ent_type
+
+cdef int fill_context(atom_t* ctxt, const StateC* st) nogil:
+    # Take care to fill every element of context!
+    # We could memset, but this makes it very easy to have broken features that
+    # make almost no impact on accuracy. If instead they're unset, the impact
+    # tends to be dramatic, so we get an obvious regression to fix...
+    fill_token(&ctxt[S2w], st.S_(2))
+    fill_token(&ctxt[S1w], st.S_(1))
+    fill_token(&ctxt[S1rw], st.R_(st.S(1), 1))
+    fill_token(&ctxt[S0lw], st.L_(st.S(0), 1))
+    fill_token(&ctxt[S0l2w], st.L_(st.S(0), 2))
+    fill_token(&ctxt[S0w], st.S_(0))
+    fill_token(&ctxt[S0r2w], st.R_(st.S(0), 2))
+    fill_token(&ctxt[S0rw], st.R_(st.S(0), 1))
+    fill_token(&ctxt[N0lw], st.L_(st.B(0), 1))
+    fill_token(&ctxt[N0l2w], st.L_(st.B(0), 2))
+    fill_token(&ctxt[N0w], st.B_(0))
+    fill_token(&ctxt[N1w], st.B_(1))
+    fill_token(&ctxt[N2w], st.B_(2))
+    fill_token(&ctxt[P1w], st.safe_get(st.B(0)-1))
+    fill_token(&ctxt[P2w], st.safe_get(st.B(0)-2))
+
+    fill_token(&ctxt[E0w], st.E_(0))
+    fill_token(&ctxt[E1w], st.E_(1))
+
+    if st.stack_depth() >= 1 and not st.eol():
+        ctxt[dist] = min_(st.B(0) - st.E(0), 5)
+    else:
+        ctxt[dist] = 0
+    ctxt[N0lv] = min_(st.n_L(st.B(0)), 5)
+    ctxt[S0lv] = min_(st.n_L(st.S(0)), 5)
+    ctxt[S0rv] = min_(st.n_R(st.S(0)), 5)
+    ctxt[S1lv] = min_(st.n_L(st.S(1)), 5)
+    ctxt[S1rv] = min_(st.n_R(st.S(1)), 5)
+
+    ctxt[S0_has_head] = 0
+    ctxt[S1_has_head] = 0
+    ctxt[S2_has_head] = 0
+    if st.stack_depth() >= 1:
+        ctxt[S0_has_head] = st.has_head(st.S(0)) + 1
+        if st.stack_depth() >= 2:
+            ctxt[S1_has_head] = st.has_head(st.S(1)) + 1
+            if st.stack_depth() >= 3:
+                ctxt[S2_has_head] = st.has_head(st.S(2)) + 1
+
+
+cdef inline int min_(int a, int b) nogil:
+    return a if a > b else b
+
+
+ner = (
+    (N0W,),
+    (P1W,),
+    (N1W,),
+    (P2W,),
+    (N2W,),
+
+    (P1W, N0W,),
+    (N0W, N1W),
+
+    (N0_prefix,),
+    (N0_suffix,),
+
+    (P1_shape,),
+    (N0_shape,),
+    (N1_shape,),
+    (P1_shape, N0_shape,),
+    (N0_shape, P1_shape,),
+    (P1_shape, N0_shape, N1_shape),
+    (N2_shape,),
+    (P2_shape,),
+
+    #(P2_norm, P1_norm, W_norm),
+    #(P1_norm, W_norm, N1_norm),
+    #(W_norm, N1_norm, N2_norm)
+
+    (P2p,),
+    (P1p,),
+    (N0p,),
+    (N1p,),
+    (N2p,),
+
+    (P1p, N0p),
+    (N0p, N1p),
+    (P2p, P1p, N0p),
+    (P1p, N0p, N1p),
+    (N0p, N1p, N2p),
+
+    (P2c,),
+    (P1c,),
+    (N0c,),
+    (N1c,),
+    (N2c,),
+
+    (P1c, N0c),
+    (N0c, N1c),
+
+    (E0W,),
+    (E0c,),
+    (E0p,),
+
+    (E0W, N0W),
+    (E0c, N0W),
+    (E0p, N0W),
+
+    (E0p, P1p, N0p),
+    (E0c, P1c, N0c),
+
+    (E0w, P1c),
+    (E0p, P1p),
+    (E0c, P1c),
+    (E0p, E1p),
+    (E0c, P1p),
+
+    (E1W,),
+    (E1c,),
+    (E1p,),
+
+    (E0W, E1W),
+    (E0W, E1p,),
+    (E0p, E1W,),
+    (E0p, E1W),
+
+    (P1_ne_iob,),
+    (P1_ne_iob, P1_ne_type),
+    (N0w, P1_ne_iob, P1_ne_type),
+
+    (N0_shape,),
+    (N1_shape,),
+    (N2_shape,),
+    (P1_shape,),
+    (P2_shape,),
+
+    (N0_prefix,),
+    (N0_suffix,),
+
+    (P1_ne_iob,),
+    (P2_ne_iob,),
+    (P1_ne_iob, P2_ne_iob),
+    (P1_ne_iob, P1_ne_type),
+    (P2_ne_iob, P2_ne_type),
+    (N0w, P1_ne_iob, P1_ne_type),
+
+    (N0w, N1w),
+)
+
+
+unigrams = (
+    (S2W, S2p),
+    (S2c6, S2p),
+
+    (S1W, S1p),
+    (S1c6, S1p),
+
+    (S0W, S0p),
+    (S0c6, S0p),
+
+    (N0W, N0p),
+    (N0p,),
+    (N0c,),
+    (N0c6, N0p),
+    (N0L,),
+
+    (N1W, N1p),
+    (N1c6, N1p),
+
+    (N2W, N2p),
+    (N2c6, N2p),
+
+    (S0r2W, S0r2p),
+    (S0r2c6, S0r2p),
+    (S0r2L,),
+
+    (S0rW, S0rp),
+    (S0rc6, S0rp),
+    (S0rL,),
+
+    (S0l2W, S0l2p),
+    (S0l2c6, S0l2p),
+    (S0l2L,),
+
+    (S0lW, S0lp),
+    (S0lc6, S0lp),
+    (S0lL,),
+
+    (N0l2W, N0l2p),
+    (N0l2c6, N0l2p),
+    (N0l2L,),
+
+    (N0lW, N0lp),
+    (N0lc6, N0lp),
+    (N0lL,),
+)
+
+
+s0_n0 = (
+    (S0W, S0p, N0W, N0p),
+    (S0c, S0p, N0c, N0p),
+    (S0c6, S0p, N0c6, N0p),
+    (S0c4, S0p, N0c4, N0p),
+    (S0p, N0p),
+    (S0W, N0p),
+    (S0p, N0W),
+    (S0W, N0c),
+    (S0c, N0W),
+    (S0p, N0c),
+    (S0c, N0p),
+    (S0W, S0rp, N0p),
+    (S0p, S0rp, N0p),
+    (S0p, N0lp, N0W),
+    (S0p, N0lp, N0p),
+    (S0L, N0p),
+    (S0p, S0rL, N0p),
+    (S0p, N0lL, N0p),
+    (S0p, S0rv, N0p),
+    (S0p, N0lv, N0p),
+    (S0c6, S0rL, S0r2L, N0p),
+    (S0p, N0lL, N0l2L, N0p),
+)
+
+
+s1_s0 = (
+    (S1p, S0p),
+    (S1p, S0p, S0_has_head),
+    (S1W, S0p),
+    (S1W, S0p, S0_has_head),
+    (S1c, S0p),
+    (S1c, S0p, S0_has_head),
+    (S1p, S1rL, S0p),
+    (S1p, S1rL, S0p, S0_has_head),
+    (S1p, S0lL, S0p),
+    (S1p, S0lL, S0p, S0_has_head),
+    (S1p, S0lL, S0l2L, S0p),
+    (S1p, S0lL, S0l2L, S0p, S0_has_head),
+    (S1L, S0L, S0W),
+    (S1L, S0L, S0p),
+    (S1p, S1L, S0L, S0p),
+    (S1p, S0p),
+)
+
+
+s1_n0 = (
+    (S1p, N0p),
+    (S1c, N0c),
+    (S1c, N0p),
+    (S1p, N0c),
+    (S1W, S1p, N0p),
+    (S1p, N0W, N0p),
+    (S1c6, S1p, N0c6, N0p),
+    (S1L, N0p),
+    (S1p, S1rL, N0p),
+    (S1p, S1rp, N0p),
+)
+
+
+s0_n1 = (
+    (S0p, N1p),
+    (S0c, N1c),
+    (S0c, N1p),
+    (S0p, N1c),
+    (S0W, S0p, N1p),
+    (S0p, N1W, N1p),
+    (S0c6, S0p, N1c6, N1p),
+    (S0L, N1p),
+    (S0p, S0rL, N1p),
+)
+
+
+n0_n1 = (
+    (N0W, N0p, N1W, N1p),
+    (N0W, N0p, N1p),
+    (N0p, N1W, N1p),
+    (N0c, N0p, N1c, N1p),
+    (N0c6, N0p, N1c6, N1p),
+    (N0c, N1c),
+    (N0p, N1c),
+)
+
+tree_shape = (
+    (dist,),
+    (S0p, S0_has_head, S1_has_head, S2_has_head),
+    (S0p, S0lv, S0rv),
+    (N0p, N0lv),
+)
+
+trigrams = (
+    (N0p, N1p, N2p),
+    (S0p, S0lp, S0l2p),
+    (S0p, S0rp, S0r2p),
+    (S0p, S1p, S2p),
+    (S1p, S0p, N0p),
+    (S0p, S0lp, N0p),
+    (S0p, N0p, N0lp),
+    (N0p, N0lp, N0l2p),
+
+    (S0W, S0p, S0rL, S0r2L),
+    (S0p, S0rL, S0r2L),
+
+    (S0W, S0p, S0lL, S0l2L),
+    (S0p, S0lL, S0l2L),
+
+    (N0W, N0p, N0lL, N0l2L),
+    (N0p, N0lL, N0l2L),
+)
+
+
+words = (
+    S2w,
+    S1w,
+    S1rw,
+    S0lw,
+    S0l2w,
+    S0w,
+    S0r2w,
+    S0rw,
+    N0lw,
+    N0l2w,
+    N0w,
+    N1w,
+    N2w,
+    P1w,
+    P2w
+)
+
+tags = (
+    S2p,
+    S1p,
+    S1rp,
+    S0lp,
+    S0l2p,
+    S0p,
+    S0r2p,
+    S0rp,
+    N0lp,
+    N0l2p,
+    N0p,
+    N1p,
+    N2p,
+    P1p,
+    P2p
+)
+
+labels = (
+    S2L,
+    S1L,
+    S1rL,
+    S0lL,
+    S0l2L,
+    S0L,
+    S0r2L,
+    S0rL,
+    N0lL,
+    N0l2L,
+    N0L,
+    N1L,
+    N2L,
+    P1L,
+    P2L
+)

spacy/syntax/parser.pxd

@@ -1,4 +1,6 @@
 from thinc.typedefs cimport atom_t
+from thinc.linear.avgtron cimport AveragedPerceptron
+from thinc.structs cimport FeatureC
 
 from .stateclass cimport StateClass
 from .arc_eager cimport TransitionSystem
@@ -8,14 +10,17 @@ from ..structs cimport TokenC
 from ._state cimport StateC
 
 
+cdef class ParserModel(AveragedPerceptron):
+    cdef int set_featuresC(self, atom_t* context, FeatureC* features,
+            const StateC* state) nogil
+ 
+
 cdef class Parser:
     cdef readonly Vocab vocab
     cdef public object model
+    cdef readonly ParserModel _model
     cdef readonly TransitionSystem moves
     cdef readonly object cfg
     cdef public object _multitasks
 
-    cdef void _parseC(self, StateC* state, 
-            const float* feat_weights, const float* bias,
-            const float* hW, const float* hb,
-            int nr_class, int nr_hidden, int nr_feat, int nr_piece) nogil
+    cdef int parseC(self, StateC* state, TokenC* tokens, int length, int nr_feat) nogil

spacy/syntax/parser.pyx

@@ -0,0 +1,515 @@
+"""
+MALT-style dependency parser
+"""
+# coding: utf-8
+# cython: infer_types=True
+from __future__ import unicode_literals
+
+from collections import Counter
+import ujson
+
+cimport cython
+cimport cython.parallel
+
+import numpy.random
+
+from cpython.ref cimport PyObject, Py_INCREF, Py_XDECREF
+from cpython.exc cimport PyErr_CheckSignals
+from libc.stdint cimport uint32_t, uint64_t
+from libc.string cimport memset, memcpy
+from libc.stdlib cimport malloc, calloc, free
+from thinc.typedefs cimport weight_t, class_t, feat_t, atom_t, hash_t
+from thinc.linear.avgtron cimport AveragedPerceptron
+from thinc.linalg cimport VecVec
+from thinc.structs cimport SparseArrayC, FeatureC, ExampleC
+from thinc.extra.eg cimport Example
+from cymem.cymem cimport Pool, Address
+from murmurhash.mrmr cimport hash64
+from preshed.maps cimport MapStruct
+from preshed.maps cimport map_get
+from thinc.extra.search cimport Beam
+
+from .._ml import link_vectors_to_models
+from . import nonproj
+from .. import util
+from . import _parse_features
+from ._parse_features cimport CONTEXT_SIZE
+from ._parse_features cimport fill_context
+from .stateclass cimport StateClass
+from ._state cimport StateC
+from .transition_system import OracleError
+from .transition_system cimport TransitionSystem, Transition
+from ..structs cimport TokenC
+from ..tokens.doc cimport Doc
+from ..strings cimport StringStore
+from ..gold cimport GoldParse
+from ..vocab cimport Vocab
+
+
+USE_FTRL = True
+DEBUG = False
+def set_debug(val):
+    global DEBUG
+    DEBUG = val
+
+
+def get_templates(name):
+    pf = _parse_features
+    if name == 'ner':
+        return pf.ner
+    elif name == 'debug':
+        return pf.unigrams
+    elif name.startswith('embed'):
+        return (pf.words, pf.tags, pf.labels)
+    else:
+        return (pf.unigrams + pf.s0_n0 + pf.s1_n0 + pf.s1_s0 + pf.s0_n1 + pf.n0_n1 + \
+                pf.tree_shape + pf.trigrams)
+
+
+cdef class ParserModel(AveragedPerceptron):
+    @property
+    def nr_templ(self):
+        return self.extracter.nr_templ
+
+    cdef int set_featuresC(self, atom_t* context, FeatureC* features,
+            const StateC* state) nogil:
+        fill_context(context, state)
+        nr_feat = self.extracter.set_features(features, context)
+        return nr_feat
+
+    def update(self, Example eg, itn=0):
+        self.time += 1
+        cdef int best = arg_max_if_gold(eg.c.scores, eg.c.costs, eg.c.nr_class)
+        cdef int guess = eg.guess
+        if guess == best or best == -1:
+            return 0.0
+        cdef FeatureC feat
+        cdef int clas
+        cdef weight_t gradient
+        for feat in eg.c.features[:eg.c.nr_feat]:
+            self.update_weight(feat.key, guess, feat.value * eg.c.costs[guess])
+            self.update_weight(feat.key, best, -feat.value * eg.c.costs[guess])
+        return eg.c.costs[guess]
+
+
+cdef class Parser:
+    """
+    Base class of the DependencyParser and EntityRecognizer.
+    """
+
+    @classmethod
+    def Model(cls, nr_class, **cfg):
+        return ParserModel(get_templates('parser')), cfg
+    
+    def __init__(self, Vocab vocab, moves=True, model=True, **cfg):
+        """Create a Parser.
+
+        vocab (Vocab): The vocabulary object. Must be shared with documents
+            to be processed. The value is set to the `.vocab` attribute.
+        moves (TransitionSystem): Defines how the parse-state is created,
+            updated and evaluated. The value is set to the .moves attribute
+            unless True (default), in which case a new instance is created with
+            `Parser.Moves()`.
+        model (object): Defines how the parse-state is created, updated and
+            evaluated. The value is set to the .model attribute unless True
+            (default), in which case a new instance is created with
+            `Parser.Model()`.
+        **cfg: Arbitrary configuration parameters. Set to the `.cfg` attribute
+        """
+        self.vocab = vocab
+        if moves is True:
+            self.moves = self.TransitionSystem(self.vocab.strings, {})
+        else:
+            self.moves = moves
+        if 'beam_width' not in cfg:
+            cfg['beam_width'] = util.env_opt('beam_width', 1)
+        if 'beam_density' not in cfg:
+            cfg['beam_density'] = util.env_opt('beam_density', 0.0)
+        if 'pretrained_dims' not in cfg:
+            cfg['pretrained_dims'] = self.vocab.vectors.data.shape[1]
+        cfg.setdefault('cnn_maxout_pieces', 3)
+        self.cfg = cfg
+        if 'actions' in self.cfg:
+            for action, labels in self.cfg.get('actions', {}).items():
+                for label in labels:
+                    self.moves.add_action(action, label)
+        self.model = model
+        if model not in (True, False, None):
+            self._model = model
+        self._multitasks = []
+
+    def __reduce__(self):
+        return (Parser, (self.vocab, self.moves, self.model), None, None)
+    
+    def __call__(self, Doc doc, beam_width=None, beam_density=None):
+        """Apply the parser or entity recognizer, setting the annotations onto
+        the `Doc` object.
+
+        doc (Doc): The document to be processed.
+        """
+        if beam_width is None:
+            beam_width = self.cfg.get('beam_width', 1)
+        if beam_density is None:
+            beam_density = self.cfg.get('beam_density', 0.0)
+        cdef Beam beam
+        if beam_width == 1:
+            states, tokvecs = self.parse_batch([doc])
+            self.set_annotations([doc], states, tensors=tokvecs)
+            return doc
+        else:
+            beams, tokvecs = self.beam_parse([doc],
+                                beam_width=beam_width,
+                                beam_density=beam_density)
+            beam = beams[0]
+            output = self.moves.get_beam_annot(beam)
+            state = StateClass.borrow(<StateC*>beam.at(0))
+            self.set_annotations([doc], [state], tensors=tokvecs)
+            _cleanup(beam)
+            return output
+
+    def parse_batch(self, docs, batch_size=1, n_threads=1):
+        cdef Pool mem = Pool()
+        doc_ptr = <TokenC**>mem.alloc(len(docs), sizeof(TokenC*))
+        lengths = <int*>mem.alloc(len(docs), sizeof(int))
+        state_ptrs = <StateC**>mem.alloc(len(docs), sizeof(StateC*))
+        cdef Doc doc
+        cdef StateClass state
+        cdef int i
+        cdef int nr_feat = self.model.nr_feat
+        states = self.moves.init_batch(docs)
+        for i, (doc, state) in enumerate(zip(docs, states)):
+            doc_ptr[i] = doc.c
+            lengths[i] = doc.length
+            state_ptrs[i] = state.c
+        cdef int status
+        for i in range(len(docs)):
+            status = self.parseC(state_ptrs[i], doc_ptr[i], lengths[i], nr_feat)
+            #if status != 0:
+            #    with gil:
+            #        raise ParserStateError(queue[i])
+            #PyErr_CheckSignals()
+        return states, None
+
+    cdef int parseC(self, StateC* state, TokenC* tokens, int length, int nr_feat) nogil:
+        cdef int nr_class = self.moves.n_moves
+        cdef int nr_atom = CONTEXT_SIZE
+        features = <FeatureC*>calloc(sizeof(FeatureC), nr_feat)
+        atoms = <atom_t*>calloc(sizeof(atom_t), CONTEXT_SIZE)
+        scores = <weight_t*>calloc(sizeof(weight_t), nr_class)
+        is_valid = <int*>calloc(sizeof(int), nr_class)
+        cdef int i
+        while not state.is_final():
+            nr_feat = self._model.set_featuresC(atoms, features, state)
+            self.moves.set_valid(is_valid, state)
+            self._model.set_scoresC(scores, features, nr_feat)
+
+            guess = VecVec.arg_max_if_true(scores, is_valid, nr_class)
+            if guess < 0:
+                return 1
+
+            action = self.moves.c[guess]
+            action.do(state, action.label)
+            memset(scores, 0, sizeof(scores[0]) * nr_class)
+            for i in range(nr_class):
+                is_valid[i] = 1
+            tokens[i] = state._sent[i]
+        free(features)
+        free(atoms)
+        free(scores)
+        free(is_valid)
+        return 0
+
+    def update(self, docs, golds, drop=0., sgd=None, losses=None):
+        states = self.moves.init_batch(docs)
+
+        cdef GoldParse gold
+        for gold in golds:
+            self.moves.preprocess_gold(gold)
+        cdef StateClass stcls
+        cdef Pool mem = Pool()
+        cdef weight_t loss = 0
+        cdef Transition action
+        cdef double dropout_rate = self.cfg.get('dropout', drop)
+        cdef FeatureC feat
+        cdef int clas
+        cdef int nr_class = self.moves.n_moves
+        features = <FeatureC*>mem.alloc(self._model.nr_templ, sizeof(FeatureC))
+        scores = <weight_t*>mem.alloc(nr_class, sizeof(weight_t))
+        is_valid = <int*>mem.alloc(nr_class, sizeof(int))
+        costs = <weight_t*>mem.alloc(nr_class, sizeof(weight_t))
+        atoms = <atom_t*>mem.alloc(CONTEXT_SIZE, sizeof(atom_t))
+        states = self.moves.init_batch(docs)
+        for stcls, gold in zip(states, golds):
+            while not stcls.is_final():
+                memset(scores, 0, sizeof(scores[0]) * nr_class)
+                memset(costs, 0, sizeof(costs[0]) * nr_class)
+                for i in range(nr_class):
+                    is_valid[i] = 1
+                nr_feat = self._model.set_featuresC(atoms, features, stcls.c)
+                dropout(features, nr_feat, dropout_rate)
+                
+                self.moves.set_costs(is_valid, costs, stcls, gold)
+                self._model.set_scoresC(scores, features, nr_feat)
+                
+                self._model.time += 1
+                guess = VecVec.arg_max_if_true(scores, is_valid, nr_class)
+                best = arg_max_if_gold(scores, costs, nr_class)
+                if guess == best or best == -1:
+                    continue
+                for feat in features[:nr_feat]:
+                    self._model.update_weight(feat.key, guess, feat.value * costs[guess])
+                    self._model.update_weight(feat.key, best, -feat.value * costs[guess])
+                loss += costs[guess]
+
+                action = self.moves.c[guess]
+                action.do(stcls.c, action.label)
+        return loss
+
+    def add_label(self, label):
+        for action in self.moves.action_types:
+            added = self.moves.add_action(action, label)
+            if added:
+                # Important that the labels be stored as a list! We need the
+                # order, or the model goes out of synch
+                self.cfg.setdefault('extra_labels', []).append(label)
+    
+    def set_annotations(self, docs, states, tensors=None):
+        cdef StateClass state
+        cdef Doc doc
+        for i, (state, doc) in enumerate(zip(states, docs)):
+            self.moves.finalize_state(state.c)
+            for j in range(doc.length):
+                doc.c[j] = state.c._sent[j]
+            if tensors is not None:
+                if isinstance(doc.tensor, numpy.ndarray) \
+                and not isinstance(tensors[i], numpy.ndarray):
+                    doc.extend_tensor(tensors[i].get())
+                else:
+                    doc.extend_tensor(tensors[i])
+            self.moves.finalize_doc(doc)
+
+            for hook in self.postprocesses:
+                for doc in docs:
+                    hook(doc)
+
+    @property
+    def move_names(self):
+        names = []
+        for i in range(self.moves.n_moves):
+            name = self.moves.move_name(self.moves.c[i].move, self.moves.c[i].label)
+            names.append(name)
+        return names
+
+    @property
+    def postprocesses(self):
+        # Available for subclasses, e.g. to deprojectivize
+        return []
+
+    def begin_training(self, gold_tuples, pipeline=None, sgd=None, **cfg):
+        if 'model' in cfg:
+            self.model = cfg['model']
+        gold_tuples = nonproj.preprocess_training_data(gold_tuples,
+                                                       label_freq_cutoff=100)
+        actions = self.moves.get_actions(gold_parses=gold_tuples)
+        for action, labels in actions.items():
+            for label in labels:
+                self.moves.add_action(action, label)
+        if self.model is True:
+            cfg['pretrained_dims'] = self.vocab.vectors_length
+            self.model, cfg = self.Model(self.moves.n_moves, **cfg)
+            self._model = self.model
+            if sgd is None:
+                sgd = self.create_optimizer()
+            self.init_multitask_objectives(gold_tuples, pipeline, sgd=sgd, **cfg)
+            link_vectors_to_models(self.vocab)
+            self.cfg.update(cfg)
+        elif sgd is None:
+            sgd = self.create_optimizer()
+        return sgd
+
+    def init_multitask_objectives(self, gold_tuples, pipeline, **cfg):
+        '''Setup models for secondary objectives, to benefit from multi-task
+        learning. This method is intended to be overridden by subclasses.
+
+        For instance, the dependency parser can benefit from sharing
+        an input representation with a label prediction model. These auxiliary
+        models are discarded after training.
+        '''
+        pass
+
+    def preprocess_gold(self, docs_golds):
+        for doc, gold in docs_golds:
+            yield doc, gold
+
+    def use_params(self, params):
+        pass
+
+
+cdef int dropout(FeatureC* feats, int nr_feat, float prob) except -1:
+    if prob <= 0 or prob >= 1.:
+        return 0
+    cdef double[::1] py_probs = numpy.random.uniform(0., 1., nr_feat)
+    cdef double* probs = &py_probs[0]
+    for i in range(nr_feat):
+        if probs[i] >= prob:
+            feats[i].value /= prob
+        else:
+            feats[i].value = 0.
+
+
+cdef class StepwiseState:
+    cdef readonly StateClass stcls
+    cdef readonly Example eg
+    cdef readonly Doc doc
+    cdef readonly GoldParse gold
+    cdef readonly Parser parser
+
+    def __init__(self, Parser parser, Doc doc, GoldParse gold=None):
+        self.parser = parser
+        self.doc = doc
+        if gold is not None:
+            self.gold = gold
+            self.parser.moves.preprocess_gold(self.gold)
+        else:
+            self.gold = GoldParse(doc)
+        self.stcls = StateClass.init(doc.c, doc.length)
+        self.parser.moves.initialize_state(self.stcls.c)
+        self.eg = Example(
+            nr_class=self.parser.moves.n_moves,
+            nr_atom=CONTEXT_SIZE,
+            nr_feat=self.parser.model.nr_feat)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self.finish()
+
+    @property
+    def is_final(self):
+        return self.stcls.is_final()
+
+    @property
+    def stack(self):
+        return self.stcls.stack
+
+    @property
+    def queue(self):
+        return self.stcls.queue
+
+    @property
+    def heads(self):
+        return [self.stcls.H(i) for i in range(self.stcls.c.length)]
+
+    @property
+    def deps(self):
+        return [self.doc.vocab.strings[self.stcls.c._sent[i].dep]
+                for i in range(self.stcls.c.length)]
+
+    @property
+    def costs(self):
+        """
+        Find the action-costs for the current state.
+        """
+        if not self.gold:
+            raise ValueError("Can't set costs: No GoldParse provided")
+        self.parser.moves.set_costs(self.eg.c.is_valid, self.eg.c.costs,
+                self.stcls, self.gold)
+        costs = {}
+        for i in range(self.parser.moves.n_moves):
+            if not self.eg.c.is_valid[i]:
+                continue
+            transition = self.parser.moves.c[i]
+            name = self.parser.moves.move_name(transition.move, transition.label)
+            costs[name] = self.eg.c.costs[i]
+        return costs
+
+    def predict(self):
+        self.eg.reset()
+        self.eg.c.nr_feat = self.parser._model.set_featuresC(self.eg.c.atoms, self.eg.c.features,
+                                                            self.stcls.c)
+        self.parser.moves.set_valid(self.eg.c.is_valid, self.stcls.c)
+        self.parser._model.set_scoresC(self.eg.c.scores,
+            self.eg.c.features, self.eg.c.nr_feat)
+
+        cdef Transition action = self.parser.moves.c[self.eg.guess]
+        return self.parser.moves.move_name(action.move, action.label)
+
+    def transition(self, action_name=None):
+        if action_name is None:
+            action_name = self.predict()
+        moves = {'S': 0, 'D': 1, 'L': 2, 'R': 3}
+        if action_name == '_':
+            action_name = self.predict()
+            action = self.parser.moves.lookup_transition(action_name)
+        elif action_name == 'L' or action_name == 'R':
+            self.predict()
+            move = moves[action_name]
+            clas = _arg_max_clas(self.eg.c.scores, move, self.parser.moves.c,
+                                 self.eg.c.nr_class)
+            action = self.parser.moves.c[clas]
+        else:
+            action = self.parser.moves.lookup_transition(action_name)
+        action.do(self.stcls.c, action.label)
+
+    def finish(self):
+        if self.stcls.is_final():
+            self.parser.moves.finalize_state(self.stcls.c)
+        self.doc.set_parse(self.stcls.c._sent)
+        self.parser.moves.finalize_doc(self.doc)
+
+
+class ParserStateError(ValueError):
+    def __init__(self, doc):
+        ValueError.__init__(self,
+            "Error analysing doc -- no valid actions available. This should "
+            "never happen, so please report the error on the issue tracker. "
+            "Here's the thread to do so --- reopen it if it's closed:\n"
+            "https://github.com/spacy-io/spaCy/issues/429\n"
+            "Please include the text that the parser failed on, which is:\n"
+            "%s" % repr(doc.text))
+
+cdef int arg_max_if_gold(const weight_t* scores, const weight_t* costs, int n) nogil:
+    cdef int best = -1
+    for i in range(n):
+        if costs[i] <= 0:
+            if best == -1 or scores[i] > scores[best]:
+                best = i
+    return best
+
+
+cdef int _arg_max_clas(const weight_t* scores, int move, const Transition* actions,
+                       int nr_class) except -1:
+    cdef weight_t score = 0
+    cdef int mode = -1
+    cdef int i
+    for i in range(nr_class):
+        if actions[i].move == move and (mode == -1 or scores[i] >= score):
+            mode = i
+            score = scores[i]
+    return mode
+
+def _cleanup(Beam beam):
+    cdef StateC* state
+    # Once parsing has finished, states in beam may not be unique. Is this
+    # correct?
+    seen = set()
+    for i in range(beam.width):
+        addr = <size_t>beam._parents[i].content
+        if addr not in seen:
+            state = <StateC*>addr
+            del state
+            seen.add(addr)
+        else:
+            print(i, addr)
+            print(seen)
+            raise Exception
+        addr = <size_t>beam._states[i].content
+        if addr not in seen:
+            state = <StateC*>addr
+            del state
+            seen.add(addr)
+        else:
+            print(i, addr)
+            print(seen)
+            raise Exception