Tmp. Working on NN NER.

bin/parser/conll_train.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import print_function
 from __future__ import division
 from __future__ import unicode_literals
 
@@ -9,6 +10,9 @@ import io
 import random
 import time
 import gzip
+import re
+import numpy
+from math import sqrt
 
 import plac
 import cProfile
@@ -20,23 +24,29 @@ from spacy.gold import GoldParse
 
 from spacy.syntax.util import Config
 from spacy.syntax.arc_eager import ArcEager
-from spacy.syntax.parser import Parser
+from spacy.syntax.parser import Parser, get_templates
+from spacy.syntax.beam_parser import BeamParser
 from spacy.scorer import Scorer
 from spacy.tagger import Tagger
+from spacy.syntax.nonproj import PseudoProjectivity
+from spacy.syntax import _parse_features as pf
 
 # Last updated for spaCy v0.97
 
 
-def read_conll(file_):
+def read_conll(file_, n=0):
     """Read a standard CoNLL/MALT-style format"""
-    sents = []
-    for sent_str in file_.read().strip().split('\n\n'):
+    text = file_.read().strip()
+    sent_strs = re.split(r'\n\s*\n', text)
+    for sent_id, sent_str in enumerate(sent_strs):
+        if not sent_str.strip():
+            continue
         ids = []
         words = []
         heads = []
         labels = []
         tags = []
-        for i, line in enumerate(sent_str.split('\n')):
+        for i, line in enumerate(sent_str.strip().split('\n')):
             word, pos_string, head_idx, label = _parse_line(line)
             words.append(word)
             if head_idx < 0:
@@ -45,10 +55,10 @@ def read_conll(file_):
             heads.append(head_idx)
             labels.append(label)
             tags.append(pos_string)
-        text = ' '.join(words)
         annot = (ids, words, tags, heads, labels, ['O'] * len(ids))
-        sents.append((None, [(annot, [])]))
-    return sents
+        yield (None, [(annot, None)])
+        if n and sent_id >= n:
+            break
 
 
 def _parse_line(line):
@@ -68,21 +78,51 @@ def _parse_line(line):
         pos = pieces[4]
         head_idx = int(pieces[6])-1
         label = pieces[7]
-    if head_idx == 0:
+    if head_idx < 0:
         label = 'ROOT'
     return word, pos, head_idx, label
 
+
+def print_words(strings, words, embeddings):
+    ids = {strings[word]: word for word in words}
+    vectors = {}
+    for key, values in embeddings[5]:
+        if key in ids:
+            vectors[strings[key]] = values
+    for word in words:
+        if word in vectors:
+            print(word, vectors[word])
+
         
 def score_model(scorer, nlp, raw_text, annot_tuples, verbose=False):
     tokens = nlp.tokenizer.tokens_from_list(annot_tuples[1])
-    nlp.tagger(tokens)
+    nlp.tagger.tag_from_strings(tokens, annot_tuples[2])
     nlp.parser(tokens)
     gold = GoldParse(tokens, annot_tuples, make_projective=False)
     scorer.score(tokens, gold, verbose=verbose, punct_labels=('--', 'p', 'punct'))
 
 
-def train(Language, gold_tuples, model_dir, n_iter=15, feat_set=u'basic', seed=0,
-          gold_preproc=False, force_gold=False):
+def score_file(nlp, loc):
+    scorer = Scorer()
+    with io.open(loc, 'r', encoding='utf8') as file_:
+        for _, sents in read_conll(file_):
+            for annot_tuples, _ in sents:
+                score_model(scorer, nlp, None, annot_tuples)
+    return scorer
+
+
+def score_sents(nlp, gold_tuples):
+    scorer = Scorer()
+    for _, sents in gold_tuples:
+        for annot_tuples, _ in sents:
+            score_model(scorer, nlp, None, annot_tuples)
+    return scorer
+
+
+def train(Language, gold_tuples, model_dir, dev_loc, n_iter=15, feat_set=u'basic',
+          width=128, depth=3,
+          learn_rate=0.001, noise=0.01, update_step='sgd_cm', regularization=0.0,
+          batch_norm=False, seed=0, gold_preproc=False, force_gold=False):
     dep_model_dir = path.join(model_dir, 'deps')
     pos_model_dir = path.join(model_dir, 'pos')
     if path.exists(dep_model_dir):
@@ -92,65 +132,128 @@ def train(Language, gold_tuples, model_dir, n_iter=15, feat_set=u'basic', seed=0
     os.mkdir(dep_model_dir)
     os.mkdir(pos_model_dir)
 
-    Config.write(dep_model_dir, 'config', features=feat_set, seed=seed,
-                 labels=ArcEager.get_labels(gold_tuples))
+    if feat_set != 'neural':
+        Config.write(dep_model_dir, 'config', feat_set=feat_set, seed=seed,
+            labels=ArcEager.get_labels(gold_tuples),
+            eta=learn_rate, rho=regularization)
+
+    else:
+        hidden_layers = [width] * depth
+        Config.write(dep_model_dir, 'config',
+                     model='neural',
+                     seed=seed,
+                     labels=ArcEager.get_labels(gold_tuples),
+                     feat_set=feat_set,
+                     hidden_layers=hidden_layers,
+                     update_step=update_step,
+                     batch_norm=batch_norm,
+                     eta=learn_rate,
+                     mu=0.9,
+                     noise=noise,
+                     rho=regularization)
 
     nlp = Language(data_dir=model_dir, tagger=False, parser=False, entity=False)
+    # Insert into vocab
+    for _, sents in gold_tuples:
+        for annot_tuples, _ in sents:
+            for word in annot_tuples[1]:
+                _ = nlp.vocab[word] 
     nlp.tagger = Tagger.blank(nlp.vocab, Tagger.default_templates())
+    #nlp.parser = BeamParser.from_dir(dep_model_dir, nlp.vocab.strings, ArcEager)
     nlp.parser = Parser.from_dir(dep_model_dir, nlp.vocab.strings, ArcEager)
+    for word in nlp.vocab:
+        word.norm = word.orth
+    
+    print(nlp.parser.model.widths)
  
-    print("Itn.\tP.Loss\tUAS\tNER F.\tTag %\tToken %")
+    print("Itn.\tP.Loss\tTrain\tDev\tnr_weight\tnr_feat")
+    last_score = 0.0
+    nr_trimmed = 0
+    eg_seen = 0
+    loss = 0
+    micro_eval = gold_tuples[:50]
     for itn in range(n_iter):
-        scorer = Scorer()
-        loss = 0
-        for _, sents in gold_tuples:
-            for annot_tuples, _ in sents:
-                if len(annot_tuples[1]) == 1:
-                    continue
+        try:
+            eg_seen = _train_epoch(nlp, gold_tuples, eg_seen, itn,
+                                   dev_loc, micro_eval)
+        except KeyboardInterrupt:
+            print("Saving model...")
+            break
+    dev_uas = score_file(nlp, dev_loc).uas
+    print("Dev before average", dev_uas)
 
-                score_model(scorer, nlp, None, annot_tuples, verbose=False)
+    nlp.parser.model.end_training()
+    nlp.parser.model.dump(path.join(model_dir, 'deps', 'model'))
+    print("Saved. Evaluating...")
+    return nlp
 
-                tokens = nlp.tokenizer.tokens_from_list(annot_tuples[1])
-                nlp.tagger(tokens)
-                gold = GoldParse(tokens, annot_tuples, make_projective=True)
-                if not gold.is_projective:
-                    raise Exception(
-                        "Non-projective sentence in training, after we should "
-                        "have enforced projectivity: %s" % annot_tuples
-                    )
- 
-                loss += nlp.parser.train(tokens, gold)
-                nlp.tagger.train(tokens, gold.tags)
-        random.shuffle(gold_tuples)
-        print('%d:\t%d\t%.3f\t%.3f\t%.3f' % (itn, loss, scorer.uas,
-                                             scorer.tags_acc, scorer.token_acc))
-    print('end training')
-    nlp.end_training(model_dir)
-    print('done')
+
+def _train_epoch(nlp, gold_tuples, eg_seen, itn, dev_loc, micro_eval):
+    random.shuffle(gold_tuples)
+    loss = 0
+    nr_trimmed = 0
+    for _, sents in gold_tuples:
+        for annot_tuples, _ in sents:
+            tokens = nlp.tokenizer.tokens_from_list(annot_tuples[1])
+            nlp.tagger.tag_from_strings(tokens, annot_tuples[2])
+            gold = GoldParse(tokens, annot_tuples)
+            loss += nlp.parser.train(tokens, gold, itn=itn)
+            eg_seen += 1
+            if eg_seen % 1000 == 0:
+                if eg_seen % 20000 == 0:
+                    dev_uas = score_file(nlp, dev_loc).uas
+                else:
+                    dev_uas = 0.0
+                train_uas = score_sents(nlp, micro_eval).uas
+                nr_upd = nlp.parser.model.time
+                nr_weight = nlp.parser.model.nr_weight
+                nr_feat = nlp.parser.model.nr_active_feat
+                print('%d,%d:\t%d\t%.3f\t%.3f\t%d\t%d' % (itn, nr_upd, int(loss),
+                                                          train_uas, dev_uas,
+                                                          nr_weight, nr_feat))
+                loss = 0
+    nlp.parser.model.learn_rate *= 0.99
+    return eg_seen
 
 
 @plac.annotations(
     train_loc=("Location of CoNLL 09 formatted training file"),
     dev_loc=("Location of CoNLL 09 formatted development file"),
     model_dir=("Location of output model directory"),
-    eval_only=("Skip training, and only evaluate", "flag", "e", bool),
     n_iter=("Number of training iterations", "option", "i", int),
+    batch_norm=("Use batch normalization and residual connections", "flag", "b"),
+    update_step=("Update step", "option", "u", str),
+    learn_rate=("Learn rate", "option", "e", float),
+    regularization=("Regularization penalty", "option", "r", float),
+    gradient_noise=("Gradient noise", "option", "W", float),
+    neural=("Use neural network?", "flag", "N"),
+    width=("Width of hidden layers", "option", "w", int),
+    depth=("Number of hidden layers", "option", "d", int),
 )
-def main(train_loc, dev_loc, model_dir, n_iter=15):
+def main(train_loc, dev_loc, model_dir, n_iter=15, neural=False, batch_norm=False,
+         width=128, depth=3, learn_rate=0.001, gradient_noise=0.0, regularization=0.0,
+         update_step='sgd_cm'):
     with io.open(train_loc, 'r', encoding='utf8') as file_:
-        train_sents = read_conll(file_)
-    if not eval_only:
-        train(English, train_sents, model_dir, n_iter=n_iter)
-    nlp = English(data_dir=model_dir)
-    dev_sents = read_conll(io.open(dev_loc, 'r', encoding='utf8'))
-    scorer = Scorer()
-    for _, sents in dev_sents:
-        for annot_tuples, _ in sents:
-            score_model(scorer, nlp, None, annot_tuples)
-    print('TOK', 100-scorer.token_acc)
+        train_sents = list(read_conll(file_))
+    # Preprocess training data here before ArcEager.get_labels() is called
+    train_sents = PseudoProjectivity.preprocess_training_data(train_sents)
+
+    nlp = train(English, train_sents, model_dir, dev_loc, n_iter=n_iter,
+                width=width, depth=depth,
+                feat_set='neural' if neural else 'basic',
+                batch_norm=batch_norm,
+                learn_rate=learn_rate,
+                regularization=regularization,
+                update_step=update_step,
+                noise=gradient_noise)
+
+    scorer = score_file(nlp, dev_loc)
+    print('TOK', scorer.token_acc)
     print('POS', scorer.tags_acc)
     print('UAS', scorer.uas)
     print('LAS', scorer.las)
+    print('nr_weight', nlp.parser.model.nr_weight)
+    print('nr_feat', nlp.parser.model.nr_active_feat)
 
 
 if __name__ == '__main__':

bin/parser/train.py

@@ -23,7 +23,8 @@ from spacy.scorer import Scorer
 from spacy.syntax.arc_eager import ArcEager
 from spacy.syntax.ner import BiluoPushDown
 from spacy.tagger import Tagger
-from spacy.syntax.parser import Parser
+from spacy.syntax.parser import Parser, get_templates
+from spacy.syntax.beam_parser import BeamParser
 from spacy.syntax.nonproj import PseudoProjectivity
 
 
@@ -51,18 +52,6 @@ def add_noise(orig, noise_level):
         return ''.join(_corrupt(c, noise_level) for c in orig)
 
 
-def score_model(scorer, nlp, raw_text, annot_tuples, verbose=False):
-    if raw_text is None:
-        tokens = nlp.tokenizer.tokens_from_list(annot_tuples[1])
-    else:
-        tokens = nlp.tokenizer(raw_text)
-    nlp.tagger(tokens)
-    nlp.entity(tokens)
-    nlp.parser(tokens)
-    gold = GoldParse(tokens, annot_tuples)
-    scorer.score(tokens, gold, verbose=verbose)
-
-
 def _merge_sents(sents):
     m_deps = [[], [], [], [], [], []]
     m_brackets = []
@@ -79,7 +68,7 @@ def _merge_sents(sents):
     return [(m_deps, m_brackets)]
 
 
-def train(Language, gold_tuples, model_dir, n_iter=15, feat_set=u'basic',
+def train(Language, gold_tuples, model_dir, dev_loc, n_iter=15, feat_set=u'basic',
           seed=0, gold_preproc=False, n_sents=0, corruption_level=0,
           beam_width=1, verbose=False,
           use_orig_arc_eager=False, pseudoprojective=False):
@@ -100,54 +89,133 @@ def train(Language, gold_tuples, model_dir, n_iter=15, feat_set=u'basic',
         # preprocess training data here before ArcEager.get_labels() is called
         gold_tuples = PseudoProjectivity.preprocess_training_data(gold_tuples)
 
-    Config.write(dep_model_dir, 'config', features=feat_set, seed=seed,
+    Config.write(dep_model_dir, 'config', feat_set=feat_set, seed=seed,
                  labels=ArcEager.get_labels(gold_tuples),
+                 rho=1e-5, eta=1.0, mu=0.9, noise=0.0,
                  beam_width=beam_width,projectivize=pseudoprojective)
-    Config.write(ner_model_dir, 'config', features='ner', seed=seed,
+    #feat_set, slots = get_templates('neural')
+    #vector_widths = [10, 10, 10]
+    #hidden_layers = [100, 100, 100]
+    #update_step = 'adam'
+    #eta = 0.001
+    #rho = 1e-4
+    #Config.write(dep_model_dir, 'config', model='neural',
+    #             seed=seed, labels=ArcEager.get_labels(gold_tuples),
+    #             feat_set=feat_set,
+    #             vector_widths=vector_widths,
+    #             slots=slots,
+    #             hidden_layers=hidden_layers,
+    #             update_step=update_step,
+    #             eta=eta,
+    #             rho=rho)
+
+
+    Config.write(ner_model_dir, 'config', feat_set='ner', seed=seed,
                  labels=BiluoPushDown.get_labels(gold_tuples),
-                 beam_width=0)
+                 beam_width=beam_width, rho=1e-8, eta=1.0, mu=0.9, noise=0.0)
 
     if n_sents > 0:
         gold_tuples = gold_tuples[:n_sents]
-
+    micro_eval = gold_tuples[:50]
     nlp = Language(data_dir=model_dir, tagger=False, parser=False, entity=False)
     nlp.tagger = Tagger.blank(nlp.vocab, Tagger.default_templates())
-    nlp.parser = Parser.from_dir(dep_model_dir, nlp.vocab.strings, ArcEager)
-    nlp.entity = Parser.from_dir(ner_model_dir, nlp.vocab.strings, BiluoPushDown)
+    if beam_width >= 2:
+        nlp.parser = Parser.from_dir(dep_model_dir, nlp.vocab.strings, ArcEager)
+        nlp.entity = BeamParser.from_dir(ner_model_dir, nlp.vocab.strings, BiluoPushDown)
+    else:
+        nlp.parser = Parser.from_dir(dep_model_dir, nlp.vocab.strings, ArcEager)
+        nlp.entity = Parser.from_dir(ner_model_dir, nlp.vocab.strings, BiluoPushDown)
+    print(nlp.parser.model.widths)
+    for raw_text, sents in gold_tuples:
+        for annot_tuples, ctnt in sents:
+            for word in annot_tuples[1]:
+                _ = nlp.vocab[word]
+    eg_seen = 0
     print("Itn.\tP.Loss\tUAS\tNER F.\tTag %\tToken %")
     for itn in range(n_iter):
-        scorer = Scorer()
-        loss = 0
-        for raw_text, sents in gold_tuples:
-            if gold_preproc:
-                raw_text = None
-            else:
-                sents = _merge_sents(sents)
-            for annot_tuples, ctnt in sents:
-                if len(annot_tuples[1]) == 1:
-                    continue
-                score_model(scorer, nlp, raw_text, annot_tuples,
-                            verbose=verbose if itn >= 2 else False)
-                if raw_text is None:
-                    words = add_noise(annot_tuples[1], corruption_level)
-                    tokens = nlp.tokenizer.tokens_from_list(words)
-                else:
-                    raw_text = add_noise(raw_text, corruption_level)
-                    tokens = nlp.tokenizer(raw_text)
-                nlp.tagger(tokens)
-                gold = GoldParse(tokens, annot_tuples)
-                if not gold.is_projective:
-                    raise Exception("Non-projective sentence in training: %s" % annot_tuples[1])
-                loss += nlp.parser.train(tokens, gold)
-                nlp.entity.train(tokens, gold)
-                nlp.tagger.train(tokens, gold.tags)
-        random.shuffle(gold_tuples)
-        print('%d:\t%d\t%.3f\t%.3f\t%.3f\t%.3f' % (itn, loss, scorer.uas, scorer.ents_f,
-                                                   scorer.tags_acc,
-                                                   scorer.token_acc))
-    print('end training')
+        try:
+            eg_seen = _train_epoch(nlp, gold_tuples, eg_seen, itn,
+                                   dev_loc, micro_eval,
+                                   gold_preproc, corruption_level)
+        except KeyboardInterrupt:
+            print("Saving model...")
+            break
+    dev_uas = score_file(nlp, dev_loc).uas
+    print("Dev before average", dev_uas)
     nlp.end_training(model_dir)
-    print('done')
+    print("Saved. Evaluating...")
+
+
+def _train_epoch(nlp, gold_tuples, eg_seen, itn, dev_loc, micro_eval,
+        gold_preproc, corruption_level):
+    random.shuffle(gold_tuples)
+    loss = 0
+    nr_trimmed = 0
+    for raw_text, sents in gold_tuples:
+        if gold_preproc:
+            raw_text = None
+        else:
+            sents = _merge_sents(sents)
+        for annot_tuples, ctnt in sents:
+            if len(annot_tuples[1]) == 1:
+                continue
+            if raw_text is None:
+                words = add_noise(annot_tuples[1], corruption_level)
+                tokens = nlp.tokenizer.tokens_from_list(words)
+            else:
+                raw_text = add_noise(raw_text, corruption_level)
+                tokens = nlp.tokenizer(raw_text)
+            nlp.tagger(tokens)
+            gold = GoldParse(tokens, annot_tuples)
+            if not gold.is_projective:
+                raise Exception("Non-projective sentence in training: %s" % annot_tuples[1])
+            loss += nlp.parser.train(tokens, gold)
+            nlp.entity.train(tokens, gold)
+            nlp.tagger.train(tokens, gold.tags)
+            
+            eg_seen += 1
+            if eg_seen % 1000 == 0:
+                scorer = score_sents(nlp, micro_eval)
+                print('%d:\t%d\t%.3f\t%.3f\t%.3f\t%.3f\t%d\t%d' % (itn, loss, scorer.uas, scorer.ents_f,
+                                                           scorer.tags_acc,
+                                                           scorer.token_acc,
+                                                           nlp.parser.model.nr_active_feat,
+                                                           nlp.entity.model.nr_active_feat))
+                loss = 0
+    #nlp.parser.model.learn_rate *= 0.99
+    scorer = score_file(nlp, dev_loc)
+    print('D:\t%d\t%.3f\t%.3f\t%.3f\t%.3f' %  (loss, scorer.uas, scorer.ents_f,
+                                               scorer.tags_acc, scorer.token_acc))
+    return eg_seen
+
+
+def score_file(nlp, loc):
+    gold_sents = read_json_file(loc, verbose=False)
+    scorer = Scorer()
+    for _, sents in gold_sents:
+        for annot_tuples, _ in sents:
+            score_model(scorer, nlp, None, annot_tuples)
+    return scorer
+
+
+def score_sents(nlp, gold_tuples):
+    scorer = Scorer()
+    for _, sents in gold_tuples:
+        for annot_tuples, _ in sents:
+            score_model(scorer, nlp, None, annot_tuples)
+    return scorer
+
+
+def score_model(scorer, nlp, raw_text, annot_tuples, verbose=False):
+    if raw_text is None:
+        tokens = nlp.tokenizer.tokens_from_list(annot_tuples[1])
+    else:
+        tokens = nlp.tokenizer(raw_text)
+    nlp.tagger(tokens)
+    nlp.entity(tokens)
+    nlp.parser(tokens)
+    gold = GoldParse(tokens, annot_tuples)
+    scorer.score(tokens, gold, verbose=verbose)
 
 
 def evaluate(Language, gold_tuples, model_dir, gold_preproc=False, verbose=False,
@@ -178,7 +246,7 @@ def evaluate(Language, gold_tuples, model_dir, gold_preproc=False, verbose=False
 
 def write_parses(Language, dev_loc, model_dir, out_loc):
     nlp = Language(data_dir=model_dir)
-    gold_tuples = read_json_file(dev_loc)
+    gold_tuples = read_json_file(dev_loc, verbose=True)
     scorer = Scorer()
     out_file = io.open(out_loc, 'w', 'utf8')
     for raw_text, sents in gold_tuples:
@@ -207,6 +275,7 @@ def write_parses(Language, dev_loc, model_dir, out_loc):
     train_loc=("Location of training file or directory"),
     dev_loc=("Location of development file or directory"),
     model_dir=("Location of output model directory",),
+    beam_width=("Parser and NER beam width", "option", "k", int),
     eval_only=("Skip training, and only evaluate", "flag", "e", bool),
     corruption_level=("Amount of noise to add to training data", "option", "c", float),
     gold_preproc=("Use gold-standard sentence boundaries in training?", "flag", "g", bool),
@@ -218,19 +287,22 @@ def write_parses(Language, dev_loc, model_dir, out_loc):
     pseudoprojective=("Use pseudo-projective parsing", "flag", "p", bool),
 )
 def main(language, train_loc, dev_loc, model_dir, n_sents=0, n_iter=15, out_loc="", verbose=False,
-         debug=False, corruption_level=0.0, gold_preproc=False, eval_only=False, pseudoprojective=False):
+         debug=False, corruption_level=0.0, beam_width=1,
+         gold_preproc=False, eval_only=False, pseudoprojective=False):
     lang = spacy.util.get_lang_class(language)
 
     if not eval_only:
-        gold_train = list(read_json_file(train_loc))
-        train(lang, gold_train, model_dir,
-              feat_set='basic' if not debug else 'debug',
+        gold_train = list(read_json_file(train_loc, verbose=True))
+        train(lang, gold_train, model_dir, dev_loc,
+              feat_set='basic', #'neural' if not debug else 'debug',
               gold_preproc=gold_preproc, n_sents=n_sents,
               corruption_level=corruption_level, n_iter=n_iter,
-              verbose=verbose,pseudoprojective=pseudoprojective)
+              verbose=verbose, pseudoprojective=pseudoprojective,
+              beam_width=beam_width)
     if out_loc:
         write_parses(lang, dev_loc, model_dir, out_loc)
-    scorer = evaluate(lang, list(read_json_file(dev_loc)),
+    print(model_dir)
+    scorer = evaluate(lang, list(read_json_file(dev_loc, verbose=True)),
                       model_dir, gold_preproc=gold_preproc, verbose=verbose)
     print('TOK', scorer.token_acc)
     print('POS', scorer.tags_acc)

bin/parser/train_ud.py

@@ -16,24 +16,86 @@ from spacy.syntax.arc_eager import ArcEager
 from spacy.syntax.parser import get_templates
 from spacy.scorer import Scorer
 import spacy.attrs
+from spacy.syntax.nonproj import PseudoProjectivity
+
+from spacy.syntax._parse_features import *
 
 from spacy.language import Language
 
-from spacy.tagger import W_orth
-
-TAGGER_TEMPLATES = (
-    (W_orth,),
-)
-
 try:
     from codecs import open
 except ImportError:
     pass
 
 
+features = [
+    (S2W,),
+    (S1W, ),
+    (S1rW,),
+    (S0lW, ),
+    (S0l2W, ),
+    (S0W, ),
+    (S0r2W, ),
+    (S0rW, ),
+    (N0l2W, ),
+    (N0lW, ),
+    (N0W, ),
+    (N1W, ),
+    (N2W, )
+]
+
+slots = [0] * len(features)
+
+features += [
+    (S2p,),
+    (S1p, ),
+    (S1rp,),
+    (S0lp,),
+    (S0l2p,),
+    (S0p, ),
+    (S0r2p, ),
+    (S0rp, ),
+    (N0l2p, ),
+    (N0lp, ),
+    (N0p, ),
+    (N1p, ),
+    (N2p, )
+]
+
+slots += [1] * (len(features) - len(slots))
+
+features += [
+    (S2L,),
+    (S1L,),
+    (S1rL,),
+    (S0lL,),
+    (S0l2L,),
+    (S0L,),
+    (S0rL,),
+    (S0r2L,),
+    (N0l2L,),
+    (N0lL,),
+]
+slots += [2] * (len(features) - len(slots))
+#
+#features += [(S2p, S1p), (S1p, S0p)]
+#slots += [3, 3]
+#features += [(S0p, N0p)]
+#slots += [4]
+#    (S0l2p, S0l2L, S0lp, S0l2L),
+#    (N0l2p, N0l2L, N0lp, N0lL),
+#    (S1p, S1rp, S1rL),
+#    (S0p, S0rp, S0rL),
+#)
+
+
+
+
 class TreebankParser(object):
     @staticmethod
-    def setup_model_dir(model_dir, labels, templates, feat_set='basic', seed=0):
+    def setup_model_dir(model_dir, labels, vector_widths=(300,), slots=(0,),
+            hidden_layers=(300, 300),
+            feat_set='basic', seed=0, update_step='sgd', eta=0.005, rho=0.0):
         dep_model_dir = path.join(model_dir, 'deps')
         pos_model_dir = path.join(model_dir, 'pos')
         if path.exists(dep_model_dir):
@@ -43,15 +105,16 @@ class TreebankParser(object):
         os.mkdir(dep_model_dir)
         os.mkdir(pos_model_dir)
 
-        Config.write(dep_model_dir, 'config', features=feat_set, seed=seed,
-                     labels=labels)
+        Config.write(dep_model_dir, 'config', model='neural', feat_set=feat_set,
+                     seed=seed, labels=labels, vector_widths=vector_widths, slots=slots,
+                     hidden_layers=hidden_layers, update_step=update_step, eta=eta, rho=rho)
 
     @classmethod
     def from_dir(cls, tag_map, model_dir):
-        vocab = Vocab(tag_map=tag_map, get_lex_attr=Language.default_lex_attrs())
+        vocab = Vocab.load(model_dir, get_lex_attr=Language.default_lex_attrs())
         vocab.get_lex_attr[spacy.attrs.LANG] = lambda _: 0
         tokenizer = Tokenizer(vocab, {}, None, None, None)
-        tagger = Tagger.blank(vocab, TAGGER_TEMPLATES)
+        tagger = Tagger.blank(vocab, Tagger.default_templates())
 
         cfg = Config.read(path.join(model_dir, 'deps'), 'config')
         parser = Parser.from_dir(path.join(model_dir, 'deps'), vocab.strings, ArcEager)
@@ -64,22 +127,14 @@ class TreebankParser(object):
         self.parser = parser
 
     def train(self, words, tags, heads, deps):
-        tokens = self.tokenizer.tokens_from_list(list(words))
-        self.tagger.train(tokens, tags)
-        
         tokens = self.tokenizer.tokens_from_list(list(words))
         ids = range(len(words))
         ner = ['O'] * len(words)
-        gold = GoldParse(tokens, ((ids, words, tags, heads, deps, ner)),
-                         make_projective=False)
-        self.tagger(tokens)
-        if gold.is_projective:
-            try:
-                self.parser.train(tokens, gold)
-            except:
-                for id_, word, head, dep in zip(ids, words, heads, deps):
-                    print(id_, word, head, dep)
-                raise
+        gold = GoldParse(tokens, ((ids, words, tags, heads, deps, ner)))
+        self.tagger.tag_from_strings(tokens, tags)
+        loss = self.parser.train(tokens, gold)
+        PseudoProjectivity.deprojectivize(tokens)
+        return loss
 
     def __call__(self, words, tags=None):
         tokens = self.tokenizer.tokens_from_list(list(words))
@@ -88,6 +143,7 @@ class TreebankParser(object):
         else:
             self.tagger.tag_from_strings(tokens, tags)
         self.parser(tokens)
+        PseudoProjectivity.deprojectivize(tokens)
         return tokens
 
     def end_training(self, data_dir):
@@ -101,8 +157,6 @@ class TreebankParser(object):
         self.vocab.dump(path.join(data_dir, 'vocab', 'lexemes.bin'))
 
 
- 
-
 def read_conllx(loc):
     with open(loc, 'r', 'utf8') as file_:
         text = file_.read()
@@ -119,8 +173,8 @@ def read_conllx(loc):
                 id_ = int(id_) - 1
                 head = (int(head) - 1) if head != '0' else id_
                 dep = 'ROOT' if dep == 'root' else dep
-                tokens.append((id_, word, tag, head, dep, 'O'))
-            tuples = zip(*tokens)
+                tokens.append([id_, word, tag, head, dep, 'O'])
+            tuples = [list(el) for el in zip(*tokens)]
             yield (None, [(tuples, [])])
 
 
@@ -134,27 +188,38 @@ def score_model(nlp, gold_docs, verbose=False):
     return scorer
 
 
-def main(train_loc, dev_loc, model_dir, tag_map_loc):
+@plac.annotations(
+    n_iter=("Number of training iterations", "option", "i", int),
+)
+def main(train_loc, dev_loc, model_dir, tag_map_loc, n_iter=10):
     with open(tag_map_loc) as file_:
         tag_map = json.loads(file_.read())
     train_sents = list(read_conllx(train_loc))
-    labels = ArcEager.get_labels(train_sents)
-    templates = get_templates('basic')
+    train_sents = PseudoProjectivity.preprocess_training_data(train_sents)
+    dev_sents = list(read_conllx(dev_loc))
 
-    TreebankParser.setup_model_dir(model_dir, labels, templates)
+    labels = ArcEager.get_labels(train_sents)
+
+    TreebankParser.setup_model_dir(model_dir, labels,
+        feat_set=features, vector_widths=(10,10,10,30,30), slots=slots,
+        hidden_layers=(100,100,100), update_step='adam')
     
     nlp = TreebankParser.from_dir(tag_map, model_dir)
+    nlp.parser.model.rho = 1e-4
+    print(nlp.parser.model.widths)
 
-    for itn in range(15):
+    for itn in range(n_iter):
+        loss = 0.0
         for _, doc_sents in train_sents:
             for (ids, words, tags, heads, deps, ner), _ in doc_sents:
-                nlp.train(words, tags, heads, deps)
+                loss += nlp.train(words, tags, heads, deps)
         random.shuffle(train_sents)
-        scorer = score_model(nlp, read_conllx(dev_loc))
-        print('%d:\t%.3f\t%.3f' % (itn, scorer.uas, scorer.tags_acc))
+        scorer = score_model(nlp, dev_sents)
+        print('%d:\t%.3f\t%.3f\t%.3f' % (itn, loss, scorer.uas, scorer.tags_acc))
+        print(nlp.parser.model.mem.size)
     nlp.end_training(model_dir)
     scorer = score_model(nlp, read_conllx(dev_loc))
-    print('%d:\t%.3f\t%.3f\t%.3f' % (itn, scorer.uas, scorer.las, scorer.tags_acc))
+    print('Dev: %.3f\t%.3f\t%.3f' % (scorer.uas, scorer.las, scorer.tags_acc))
  
 
 if __name__ == '__main__':

lang_data/en/morphs.json

@@ -55,5 +55,8 @@
     "VBD": {
         "was":  {"L": "be", "VerbForm": "Fin", "Tense": "Past", "Number": "Sing"},
         "were": {"L": "be", "VerbForm": "Fin", "Tense": "Past", "Number": "Plur"}
+    },
+    "VBG": {
+        "coping": {"L": "cope", "VerbForm": "ger"}
     }
 }

setup.py

@@ -50,7 +50,9 @@ MOD_NAMES = [
     'spacy.syntax.stateclass',
     'spacy.syntax._state',
     'spacy.tokenizer',
+    'spacy.syntax._neural',
     'spacy.syntax.parser',
+    'spacy.syntax.beam_parser',
     'spacy.syntax.nonproj',
     'spacy.syntax.transition_system',
     'spacy.syntax.arc_eager',

spacy/gold.pyx

@@ -1,3 +1,4 @@
+# cython: profile=True
 import numpy
 import io
 import json
@@ -264,13 +265,3 @@ cdef class GoldParse:
 
 def is_punct_label(label):
     return label == 'P' or label.lower() == 'punct'
-
-
-
-
-
-
-
-
-
-

spacy/syntax/_neural.pxd

@@ -0,0 +1,12 @@
+from thinc.linear.avgtron cimport AveragedPerceptron
+from thinc.neural.nn cimport NeuralNet
+from thinc.linear.features cimport ConjunctionExtracter
+from thinc.structs cimport NeuralNetC, FeatureC
+
+
+cdef class ParserNeuralNet(NeuralNet):
+    cdef ConjunctionExtracter extracter
+
+
+cdef class ParserPerceptron(AveragedPerceptron):
+    pass

spacy/syntax/_neural.pyx

@@ -0,0 +1,328 @@
+# cython: infer_types=True
+# cython: profile=True
+from libc.stdint cimport uint64_t
+from libc.string cimport memcpy, memset
+from libc.math cimport sqrt
+
+from cymem.cymem cimport Pool, Address
+from murmurhash.mrmr cimport hash64
+
+from thinc.typedefs cimport weight_t, class_t, feat_t, atom_t, hash_t, idx_t
+from thinc.linear.avgtron cimport AveragedPerceptron
+from thinc.linalg cimport VecVec
+from thinc.structs cimport NeuralNetC, SparseArrayC, ExampleC
+from thinc.structs cimport FeatureC
+from thinc.extra.eg cimport Example
+from thinc.neural.forward cimport softmax
+
+from preshed.maps cimport map_get
+from preshed.maps cimport MapStruct
+
+from ..structs cimport TokenC
+from ._state cimport StateC
+from ._parse_features cimport fill_context
+from ._parse_features cimport CONTEXT_SIZE
+from ._parse_features cimport fill_context
+from ._parse_features import ner as ner_templates
+from ._parse_features cimport *
+from .transition_system cimport TransitionSystem
+from ..tokens.doc cimport Doc
+
+
+cdef class ParserPerceptron(AveragedPerceptron):
+    @property
+    def widths(self):
+        return (self.extracter.nr_templ,)
+
+    def update(self, Example eg, loss='regression'):
+        self.time += 1
+        best = eg.best
+        guess = eg.guess
+        assert best >= 0, best
+        assert guess >= 0, guess
+        d_losses = {}
+        if loss == 'regression':
+            # Does regression on negative cost. Sort of cute?
+            # Clip to guess and best, to keep gradient sparse.
+            d_losses[guess] = -2 * (-eg.c.costs[guess] - eg.c.scores[guess])
+            d_losses[best] = -2 * (-eg.c.costs[best] - eg.c.scores[best])
+            #for i in range(eg.c.nr_class):
+            #    if eg.c.is_valid[i] \
+            #    and eg.c.scores[i] >= eg.c.scores[best]:
+            #        d_losses[i] = -2 * (-eg.c.costs[i] - eg.c.scores[i])
+        elif loss == 'nll':
+            # Clip to guess and best, to keep gradient sparse.
+            if eg.c.scores[guess] == 0.0:
+                d_losses[guess] = 1.0
+                d_losses[best] = -1.0
+            else:
+                softmax(eg.c.scores, eg.c.nr_class)
+                for i in range(eg.c.nr_class):
+                    if eg.c.is_valid[i] \
+                    and eg.c.scores[i] >= eg.c.scores[best]:
+                        d_losses[i] = eg.c.scores[i] - (eg.c.costs[i] <= 0)
+        elif loss == 'hinge':
+            for i in range(eg.c.nr_class):
+                if eg.c.is_valid[i] \
+                and eg.c.costs[i] > 0 \
+                and eg.c.scores[i] > (eg.c.scores[best]-1):
+                    margin = eg.c.scores[i] - (eg.c.scores[best] - 1)
+                    d_losses[i] = margin
+                    d_losses[best] = min(-margin, d_losses.get(best, 0.0))
+        elif loss == 'perceptron':
+            if guess != best:
+                d_losses = {best: -1.0, guess: 1.0}
+        step = 0.0
+        i = 0
+        for clas, d_loss in sorted(d_losses.items()):
+            for feat in eg.c.features[:eg.c.nr_feat]:
+                self.update_weight(feat.key, clas, feat.value * d_loss)
+                i += 1
+        #self.total_L1 += self.l1_penalty *  self.learn_rate
+        return sum(map(abs, d_losses.values()))
+
+    cdef int set_featuresC(self, FeatureC* feats, const void* _state) nogil: 
+        cdef atom_t[CONTEXT_SIZE] context
+        memset(context, 0, sizeof(context))
+        state = <const StateC*>_state
+        fill_context(context, state)
+        return self.extracter.set_features(feats, context)
+
+    def _update_from_history(self, TransitionSystem moves, Doc doc, history, weight_t grad):
+        cdef Pool mem = Pool()
+        features = <FeatureC*>mem.alloc(self.nr_feat, sizeof(FeatureC))
+        
+        cdef StateClass stcls = StateClass.init(doc.c, doc.length)
+        moves.initialize_state(stcls.c)
+
+        cdef class_t clas
+        self.time += 1
+        for clas in history:
+            nr_feat = self.set_featuresC(features, stcls.c)
+            for feat in features[:nr_feat]:
+                self.update_weight(feat.key, clas, feat.value * grad)
+            moves.c[clas].do(stcls.c, moves.c[clas].label)
+ 
+
+cdef class ParserNeuralNet(NeuralNet):
+    def __init__(self, shape, **kwargs):
+        if kwargs.get('feat_set', 'parser') == 'parser':
+            vector_widths = [4] * 76
+            slots =  [0, 1, 2, 3] # S0
+            slots += [4, 5, 6, 7] # S1
+            slots += [8, 9, 10, 11] # S2
+            slots += [12, 13, 14, 15] # S3+
+            slots += [16, 17, 18, 19] # B0
+            slots += [20, 21, 22, 23] # B1
+            slots += [24, 25, 26, 27] # B2
+            slots += [28, 29, 30, 31] # B3+
+            slots += [32, 33, 34, 35] * 2 # S0l, S0r
+            slots += [36, 37, 38, 39] * 2 # B0l, B0r
+            slots += [40, 41, 42, 43] * 2 # S1l, S1r
+            slots += [44, 45, 46, 47] * 2 # S2l, S2r
+            slots += [48, 49, 50, 51, 52, 53, 54, 55]
+            slots += [53, 54, 55, 56]
+            self.extracter = None
+        else:
+            templates = ner_templates
+            vector_widths = [4] * len(templates)
+            slots = list(range(templates))
+            self.extracter = ConjunctionExtracter(templates)
+ 
+        input_length = sum(vector_widths[slot] for slot in slots)
+        widths = [input_length] + shape
+        NeuralNet.__init__(self, widths, embed=(vector_widths, slots), **kwargs)
+
+    @property
+    def nr_feat(self):
+        if self.extracter is None:
+            return 2000
+        else:
+            return self.extracter.nr_feat
+
+    cdef int set_featuresC(self, FeatureC* feats, const void* _state) nogil: 
+        cdef atom_t[CONTEXT_SIZE] context
+        state = <const StateC*>_state
+        if self.extracter is not None:
+            fill_context(context, state)
+            return self.extracter.set_features(feats, context)
+        memset(feats, 0, 2000 * sizeof(FeatureC))
+        start = feats
+
+        feats = _add_token(feats, 0, state.S_(0), 1.0)
+        feats = _add_token(feats, 4, state.S_(1), 1.0)
+        feats = _add_token(feats, 8, state.S_(2), 1.0)
+        # Rest of the stack, with exponential decay
+        for i in range(3, state.stack_depth()):
+            feats = _add_token(feats, 12, state.S_(i), 1.0 * 0.5**(i-2))
+        feats = _add_token(feats, 16, state.B_(0), 1.0)
+        feats = _add_token(feats, 20, state.B_(1), 1.0)
+        feats = _add_token(feats, 24, state.B_(2), 1.0)
+        # Rest of the buffer, with exponential decay
+        for i in range(3, min(8, state.buffer_length())):
+            feats = _add_token(feats, 28, state.B_(i), 1.0 * 0.5**(i-2))
+        feats = _add_subtree(feats, 32, state, state.S(0))
+        feats = _add_subtree(feats, 40, state, state.B(0))
+        feats = _add_subtree(feats, 48, state, state.S(1))
+        feats = _add_subtree(feats, 56, state, state.S(2))
+        feats = _add_pos_bigram(feats, 64, state.S_(0), state.B_(0))
+        feats = _add_pos_bigram(feats, 65, state.S_(1), state.S_(0))
+        feats = _add_pos_bigram(feats, 66, state.S_(1), state.B_(0))
+        feats = _add_pos_bigram(feats, 67, state.S_(0), state.B_(1))
+        feats = _add_pos_bigram(feats, 68, state.S_(0), state.R_(state.S(0), 1))
+        feats = _add_pos_bigram(feats, 69, state.S_(0), state.R_(state.S(0), 2))
+        feats = _add_pos_bigram(feats, 70, state.S_(0), state.L_(state.S(0), 1))
+        feats = _add_pos_bigram(feats, 71, state.S_(0), state.L_(state.S(0), 2))
+        feats = _add_pos_trigram(feats, 72, state.S_(1), state.S_(0), state.B_(0))
+        feats = _add_pos_trigram(feats, 73, state.S_(0), state.B_(0), state.B_(1))
+        feats = _add_pos_trigram(feats, 74, state.S_(0), state.R_(state.S(0), 1),
+                                 state.R_(state.S(0), 2))
+        feats = _add_pos_trigram(feats, 75, state.S_(0), state.L_(state.S(0), 1),
+                                 state.L_(state.S(0), 2))
+        return feats - start
+
+    #cdef void _set_delta_lossC(self, weight_t* delta_loss,
+    #        const weight_t* cost, const weight_t* scores) nogil:
+    #    for i in range(self.c.widths[self.c.nr_layer-1]):
+    #        delta_loss[i] = cost[i]
+
+    #cdef void _softmaxC(self, weight_t* out) nogil:
+    #    pass
+    
+    cdef void dropoutC(self, FeatureC* feats, weight_t drop_prob,
+            int nr_feat) nogil:
+        pass
+    
+    def _update_from_history(self, TransitionSystem moves, Doc doc, history, weight_t grad):
+        cdef Pool mem = Pool()
+        features = <FeatureC*>mem.alloc(self.nr_feat, sizeof(FeatureC))
+        is_valid = <int*>mem.alloc(moves.n_moves, sizeof(int))
+        costs = <weight_t*>mem.alloc(moves.n_moves, sizeof(weight_t))
+ 
+        stcls = StateClass.init(doc.c, doc.length)
+        moves.initialize_state(stcls.c)
+        cdef uint64_t[2] key
+        key[0] = hash64(doc.c, sizeof(TokenC) * doc.length, 0)
+        key[1] = 0
+        cdef uint64_t clas
+        for clas in history:
+            memset(costs, 0, moves.n_moves * sizeof(costs[0]))
+            for i in range(moves.n_moves):
+                is_valid[i] = 1
+            nr_feat = self.set_featuresC(features, stcls.c)
+            moves.set_valid(is_valid, stcls.c)
+            # Update with a sparse gradient: everything's 0, except our class.
+            # Remember, this is a component of the global update. It's not our
+            # "job" here to think about the other beam candidates. We just want
+            # to work on this sequence. However, other beam candidates will
+            # have gradients that refer to the same state.
+            # We therefore have a key that indicates the current sequence, so that
+            # the model can merge updates that refer to the same state together,
+            # by summing their gradients.
+            costs[clas] = grad
+            self.updateC(features,
+                nr_feat, costs, is_valid, False, key[0])
+            moves.c[clas].do(stcls.c, moves.c[clas].label)
+            # Build a hash of the state sequence.
+            # Position 0 represents the previous sequence, position 1 the new class.
+            # So we want to do:
+            # key.prev = hash((key.prev, key.new))
+            # key.new = clas
+            key[1] = clas
+            key[0] = hash64(key, sizeof(key), 0)
+
+
+cdef inline FeatureC* _add_token(FeatureC* feats,
+        int slot, const TokenC* token, weight_t value) nogil:
+    # Word
+    feats.i = slot
+    feats.key = token.lex.norm
+    feats.value = value
+    feats += 1
+    # POS tag
+    feats.i = slot+1
+    feats.key = token.tag
+    feats.value = value
+    feats += 1
+    # Dependency label 
+    feats.i = slot+2
+    feats.key = token.dep
+    feats.value = value
+    feats += 1
+    # Word, label, tag
+    feats.i = slot+3
+    cdef uint64_t key[3]
+    key[0] = token.lex.cluster
+    key[1] = token.tag
+    key[2] = token.dep
+    feats.key = hash64(key, sizeof(key), 0)
+    feats.value = value
+    feats += 1
+    return feats
+
+
+cdef inline FeatureC* _add_characters(FeatureC* feats,
+        int slot, uint64_t* chars, int length, weight_t value) with gil:
+    nr_start_chars = 4
+    nr_end_chars = 4
+    for i in range(min(nr_start_chars, length)):
+        feats.i = slot
+        feats.key = chars[i]
+        feats.value = value
+        feats += 1
+        slot += 1
+    for _ in range(length, nr_start_chars):
+        feats.i = slot
+        feats.key = 0
+        feats.value = 0
+        feats += 1
+        slot += 1
+    for i in range(min(nr_end_chars, length)):
+        feats.i = slot
+        feats.key = chars[(length-nr_end_chars)+i]
+        feats.value = value
+        feats += 1
+        slot += 1
+    for _ in range(length, nr_start_chars):
+        feats.i = slot
+        feats.key = 0
+        feats.value = 0
+        feats += 1
+        slot += 1
+    return feats
+ 
+
+cdef inline FeatureC* _add_subtree(FeatureC* feats, int slot, const StateC* state, int t) nogil:
+    value = 1.0
+    for i in range(state.n_R(t)):
+        feats = _add_token(feats, slot, state.R_(t, i+1), value)
+        value *= 0.5
+    slot += 4
+    value = 1.0
+    for i in range(state.n_L(t)):
+        feats = _add_token(feats, slot, state.L_(t, i+1), value)
+        value *= 0.5
+    return feats
+
+
+cdef inline FeatureC* _add_pos_bigram(FeatureC* feat, int slot,
+        const TokenC* t1, const TokenC* t2) nogil:
+    cdef uint64_t[2] key
+    key[0] = t1.tag
+    key[1] = t2.tag
+    feat.i = slot
+    feat.key = hash64(key, sizeof(key), slot)
+    feat.value = 1.0
+    return feat+1
+ 
+
+cdef inline FeatureC* _add_pos_trigram(FeatureC* feat, int slot,
+        const TokenC* t1, const TokenC* t2, const TokenC* t3) nogil:
+    cdef uint64_t[3] key
+    key[0] = t1.tag
+    key[1] = t2.tag
+    key[2] = t3.tag
+    feat.i = slot
+    feat.key = hash64(key, sizeof(key), slot)
+    feat.value = 1.0
+    return feat+1

spacy/syntax/_parse_features.pyx

@@ -35,8 +35,8 @@ cdef inline void fill_token(atom_t* context, const TokenC* token) nogil:
         context[11] = 0
         context[12] = 0
     else:
-        context[0] = token.lex.orth
-        context[1] = token.lemma
+        context[0] = token.lex.norm
+        context[1] = token.lex.norm
         context[2] = token.tag
         context[3] = token.lex.cluster
         # We've read in the string little-endian, so now we can take & (2**n)-1
@@ -366,27 +366,26 @@ trigrams = (
 
 
 words = (
-    S2w,
-    S1w,
-    S1rw,
-    S0lw,
-    S0l2w,
-    S0w,
-    S0r2w,
-    S0rw,
-    N0lw,
-    N0l2w,
-    N0w,
-    N1w,
-    N2w,
-    P1w,
-    P2w
+    S2W,
+    S1W,
+    S1rW,
+    S0lW,
+    S0l2W,
+    S0W,
+    S0r2W,
+    S0rW,
+    N0lW,
+    N0l2W,
+    N0W,
+    N1W,
+    N2W,
+    P1W,
+    P2W
 )
 
 tags = (
     S2p,
     S1p,
-    S1rp,
     S0lp,
     S0l2p,
     S0p,
@@ -404,7 +403,6 @@ tags = (
 labels = (
     S2L,
     S1L,
-    S1rL,
     S0lL,
     S0l2L,
     S0L,
@@ -412,9 +410,88 @@ labels = (
     S0rL,
     N0lL,
     N0l2L,
-    N0L,
-    N1L,
-    N2L,
-    P1L,
-    P2L
 )
+
+core_words = (
+    S2w,
+    S1w,
+    S0lw,
+    S0l2w,
+    S0w,
+    S0rw,
+    S0r2w,
+    N0lw,
+    N0l2w,
+    N0w,
+    N1w,
+    N2w,
+)
+
+
+core_shapes = (
+    S2_shape,
+    S1_shape,
+    S0l_shape,
+    S0l2_shape,
+    S0_shape,
+    S0r_shape,
+    S0r2_shape,
+    N0l_shape,
+    N0l2_shape,
+    N0_shape,
+    N1_shape,
+    N2_shape,
+)
+
+
+core_clusters = (
+    S2c,
+    S1c,
+    S0lc,
+    S0l2c,
+    S0c,
+    S0rc,
+    S0r2c,
+    N0lc,
+    N0l2c,
+    N0c,
+    N1c,
+    N2c,
+)
+
+
+
+core_tags = (
+    S2p,
+    S1p,
+    S0lp,
+    S0l2p,
+    S0p,
+    S0r2p,
+    S0rp,
+    N0lp,
+    N0l2p,
+    N0p,
+    N1p,
+    N2p,
+)
+
+core_labels = (
+    S2L,
+    S1L,
+    S0lL,
+    S0l2L,
+    S0L,
+    S0r2L,
+    S0rL,
+    N0lL,
+    N0l2L,
+)
+
+valencies = (
+    (N0lv,),
+    (S0lv,),
+    (S0rv,),
+    (S1lv,),
+    (S1rv,),
+) 

spacy/syntax/_state.pxd

@@ -1,6 +1,9 @@
 from libc.string cimport memcpy, memset
 from libc.stdlib cimport malloc, calloc, free
-from libc.stdint cimport uint32_t
+from libc.stdint cimport uint32_t, uint64_t
+
+from murmurhash.mrmr cimport hash64
+
 from ..vocab cimport EMPTY_LEXEME
 from ..structs cimport TokenC, Entity
 from ..lexeme cimport Lexeme
@@ -201,6 +204,21 @@ cdef cppclass StateC:
         else:
             return this.length - this._b_i
 
+    uint64_t hash() nogil const:
+        cdef TokenC[11] sig
+        sig[0] = this.S_(2)[0]
+        sig[1] = this.S_(1)[0]
+        sig[2] = this.R_(this.S(1), 1)[0]
+        sig[3] = this.L_(this.S(0), 1)[0]
+        sig[4] = this.L_(this.S(0), 2)[0]
+        sig[5] = this.S_(0)[0]
+        sig[6] = this.R_(this.S(0), 2)[0]
+        sig[7] = this.R_(this.S(0), 1)[0]
+        sig[8] = this.B_(0)[0]
+        sig[9] = this.E_(0)[0]
+        sig[10] = this.E_(1)[0]
+        return hash64(sig, sizeof(sig), this._s_i)
+
     void push() nogil:
         if this.B(0) != -1:
             this._stack[this._s_i] = this.B(0)
@@ -290,6 +308,8 @@ cdef cppclass StateC:
         memcpy(this._stack, src._stack, this.length * sizeof(int))
         memcpy(this._buffer, src._buffer, this.length * sizeof(int))
         memcpy(this._ents, src._ents, this.length * sizeof(Entity))
+        memcpy(this.shifted, src.shifted, this.length * sizeof(this.shifted[0]))
+        this.length = src.length
         this._b_i = src._b_i
         this._s_i = src._s_i
         this._e_i = src._e_i

spacy/syntax/arc_eager.pyx

@@ -1,3 +1,6 @@
+# cython: profile=True
+# cython: cdivision=True
+# cython: infer_types=True
 from __future__ import unicode_literals
 
 import ctypes
@@ -24,6 +27,7 @@ from .nonproj import PseudoProjectivity
 DEF NON_MONOTONIC = True
 DEF USE_BREAK = True
 
+
 cdef weight_t MIN_SCORE = -90000
 
 # Break transition from here
@@ -65,10 +69,12 @@ cdef weight_t push_cost(StateClass stcls, const GoldParseC* gold, int target) no
 cdef weight_t pop_cost(StateClass stcls, const GoldParseC* gold, int target) nogil:
     cdef weight_t cost = 0
     cdef int i, B_i
+    # Count number of words in buffer with deendencies to/from the target.
     for i in range(stcls.buffer_length()):
         B_i = stcls.B(i)
         cost += gold.heads[B_i] == target
         cost += gold.heads[target] == B_i
+        # TODO: Should re-examine this for German --- it assumes projectivity.
         if gold.heads[B_i] == B_i or gold.heads[B_i] < target:
             break
     if Break.is_valid(stcls.c, -1) and Break.move_cost(stcls, gold) == 0:
@@ -154,7 +160,18 @@ cdef class Reduce:
 
     @staticmethod
     cdef inline weight_t move_cost(StateClass st, const GoldParseC* gold) nogil:
-        return pop_cost(st, gold, st.S(0))
+        cost = pop_cost(st, gold, st.S(0))
+        if not st.has_head(st.S(0)):
+            # Decrement cost for the arcs we save
+            for i in range(1, st.stack_depth()):
+                S_i = st.S(i)
+                if gold.heads[st.S(0)] == S_i:
+                    cost -= 1
+                if gold.heads[S_i] == st.S(0):
+                    cost -= 1
+            if Break.is_valid(st.c, -1) and Break.move_cost(st, gold) == 0:
+                cost -= 1
+        return cost
 
     @staticmethod
     cdef inline weight_t label_cost(StateClass s, const GoldParseC* gold, int label) nogil:
@@ -180,7 +197,8 @@ cdef class LeftArc:
     cdef inline weight_t move_cost(StateClass s, const GoldParseC* gold) nogil:
         cdef weight_t cost = 0
         if arc_is_gold(gold, s.B(0), s.S(0)):
-            return 0
+            # Have a negative cost if we 'recover' from the wrong dependency
+            return 0 if not s.has_head(s.S(0)) else -1
         else:
             # Account for deps we might lose between S0 and stack
             if not s.has_head(s.S(0)):
@@ -407,7 +425,7 @@ cdef class ArcEager(TransitionSystem):
         cdef move_cost_func_t[N_MOVES] move_cost_funcs
         cdef weight_t[N_MOVES] move_costs
         for i in range(N_MOVES):
-            move_costs[i] = -1
+            move_costs[i] = 9000
         move_cost_funcs[SHIFT] = Shift.move_cost
         move_cost_funcs[REDUCE] = Reduce.move_cost
         move_cost_funcs[LEFT] = LeftArc.move_cost
@@ -429,11 +447,18 @@ cdef class ArcEager(TransitionSystem):
                 is_valid[i] = True
                 move = self.c[i].move
                 label = self.c[i].label
-                if move_costs[move] == -1:
+                if move_costs[move] == 9000:
                     move_costs[move] = move_cost_funcs[move](stcls, &gold.c)
                 costs[i] = move_costs[move] + label_cost_funcs[move](stcls, &gold.c, label)
-                n_gold += costs[i] == 0
+                n_gold += costs[i] <= 0
             else:
                 is_valid[i] = False
                 costs[i] = 9000
-        assert n_gold >= 1
+        if n_gold < 1:
+            for annot in gold.orig_annot:
+                print(annot)
+            print([move_costs[i] for i in range(N_MOVES)])
+            print(gold.orig_annot[stcls.S(0)][1], gold.orig_annot[stcls.B(0)][1])
+            print(gold.heads[stcls.S(0)], gold.heads[stcls.B(0)])
+            print(gold.labels[stcls.S(0)], gold.labels[stcls.B(0)])
+            raise Exception("No gold moves")

spacy/syntax/beam_parser.pyx

@@ -0,0 +1,261 @@
+# cython: profile=True
+# cython: experimental_cpp_class_def=True
+# cython: cdivision=True
+# cython: infer_types=True
+"""
+MALT-style dependency parser
+"""
+from __future__ import unicode_literals
+cimport cython
+
+from cpython.ref cimport PyObject, Py_INCREF, Py_XDECREF
+
+from libc.stdint cimport uint32_t, uint64_t
+from libc.string cimport memset, memcpy
+from libc.stdlib cimport rand
+from libc.math cimport log, exp, isnan, isinf
+import random
+import os.path
+from os import path
+import shutil
+import json
+import math
+
+from cymem.cymem cimport Pool, Address
+from murmurhash.mrmr cimport real_hash64 as hash64
+from thinc.typedefs cimport weight_t, class_t, feat_t, atom_t, hash_t
+
+
+from util import Config
+
+from thinc.linear.features cimport ConjunctionExtracter
+from thinc.structs cimport FeatureC, ExampleC
+
+from thinc.extra.search cimport Beam
+from thinc.extra.search cimport MaxViolation
+from thinc.extra.eg cimport Example
+from thinc.extra.mb cimport Minibatch
+
+from ..structs cimport TokenC
+
+from ..tokens.doc cimport Doc
+from ..strings cimport StringStore
+
+from .transition_system cimport TransitionSystem, Transition
+
+from ..gold cimport GoldParse
+
+from . import _parse_features
+from ._parse_features cimport CONTEXT_SIZE
+from ._parse_features cimport fill_context
+from .stateclass cimport StateClass
+from .parser cimport Parser
+from ._neural cimport ParserPerceptron
+from ._neural cimport ParserNeuralNet
+
+
+DEBUG = False
+def set_debug(val):
+    global DEBUG
+    DEBUG = val
+
+
+def get_templates(name):
+    pf = _parse_features
+    if name == 'ner':
+        return pf.ner
+    elif name == 'debug':
+        return pf.unigrams
+    else:
+        return (pf.unigrams + pf.s0_n0 + pf.s1_n0 + pf.s1_s0 + pf.s0_n1 + pf.n0_n1 + \
+                pf.tree_shape + pf.trigrams)
+
+
+cdef int BEAM_WIDTH = 16
+cdef weight_t BEAM_DENSITY = 0.01
+
+cdef class BeamParser(Parser):
+    cdef public int beam_width
+    cdef public weight_t beam_density
+
+    def __init__(self, *args, **kwargs):
+        self.beam_width = kwargs.get('beam_width', BEAM_WIDTH)
+        self.beam_density = kwargs.get('beam_density', BEAM_DENSITY)
+        Parser.__init__(self, *args, **kwargs)
+
+    cdef int parseC(self, TokenC* tokens, int length, int nr_feat, int nr_class) with gil:
+        self._parseC(tokens, length, nr_feat, nr_class)
+
+    cdef int _parseC(self, TokenC* tokens, int length, int nr_feat, int nr_class) except -1:
+        cdef Beam beam = Beam(self.moves.n_moves, self.beam_width, min_density=self.beam_density)
+        beam.initialize(_init_state, length, tokens)
+        beam.check_done(_check_final_state, NULL)
+        if beam.is_done:
+            _cleanup(beam)
+            return 0
+        while not beam.is_done:
+            self._advance_beam(beam, None, False)
+        state = <StateClass>beam.at(0)
+        self.moves.finalize_state(state.c)
+        for i in range(length):
+            tokens[i] = state.c._sent[i]
+        _cleanup(beam)
+
+    def train(self, Doc tokens, GoldParse gold_parse, itn=0):
+        self.moves.preprocess_gold(gold_parse)
+        cdef Beam pred = Beam(self.moves.n_moves, self.beam_width)
+        pred.initialize(_init_state, tokens.length, tokens.c)
+        pred.check_done(_check_final_state, NULL)
+        
+        cdef Beam gold = Beam(self.moves.n_moves, self.beam_width, min_density=0.0)
+        gold.initialize(_init_state, tokens.length, tokens.c)
+        gold.check_done(_check_final_state, NULL)
+        violn = MaxViolation()
+        while not pred.is_done and not gold.is_done:
+            # We search separately here, to allow for ambiguity in the gold parse.
+            self._advance_beam(pred, gold_parse, False)
+            self._advance_beam(gold, gold_parse, True)
+            violn.check_crf(pred, gold)
+            if pred.loss > 0 and pred.min_score > (gold.score + self.model.time):
+                break
+        else:
+            # The non-monotonic oracle makes it difficult to ensure final costs are
+            # correct. Therefore do final correction
+            for i in range(pred.size):
+                if is_gold(<StateClass>pred.at(i), gold_parse, self.moves.strings):
+                    pred._states[i].loss = 0.0
+                elif pred._states[i].loss == 0.0:
+                    pred._states[i].loss = 1.0
+            violn.check_crf(pred, gold)
+        assert pred.size >= 1
+        assert gold.size >= 1
+        #_check_train_integrity(pred, gold, gold_parse, self.moves)
+        histories = zip(violn.p_probs, violn.p_hist) + zip(violn.g_probs, violn.g_hist)
+        min_grad = 0.001 ** (itn+1)
+        histories = [(grad, hist) for grad, hist in histories if abs(grad) >= min_grad]
+        random.shuffle(histories)
+        for grad, hist in histories:
+            assert not math.isnan(grad) and not math.isinf(grad), hist
+            self.model._update_from_history(self.moves, tokens, hist, grad)
+        _cleanup(pred)
+        _cleanup(gold)
+        return pred.loss
+    
+    def _advance_beam(self, Beam beam, GoldParse gold, bint follow_gold):
+        cdef Pool mem = Pool()
+        features = <FeatureC*>mem.alloc(self.model.nr_feat, sizeof(FeatureC))
+        if isinstance(self.model, ParserNeuralNet):
+            mb = Minibatch(self.model.widths, beam.size)
+            for i in range(beam.size):
+                stcls = <StateClass>beam.at(i)
+                if stcls.c.is_final():
+                    nr_feat = 0
+                else:
+                    nr_feat = self.model.set_featuresC(features, stcls.c)
+                    self.moves.set_valid(beam.is_valid[i], stcls.c)
+                mb.c.push_back(features, nr_feat, beam.costs[i], beam.is_valid[i], 0)
+            self.model(mb)
+            for i in range(beam.size):
+                memcpy(beam.scores[i], mb.c.scores(i), mb.c.nr_out() * sizeof(beam.scores[i][0]))
+        else:
+            for i in range(beam.size):
+                stcls = <StateClass>beam.at(i)
+                if not stcls.is_final():
+                    nr_feat = self.model.set_featuresC(features, stcls.c)
+                    self.moves.set_valid(beam.is_valid[i], stcls.c)
+                    self.model.set_scoresC(beam.scores[i], features, nr_feat)
+        if gold is not None:
+            for i in range(beam.size):
+                stcls = <StateClass>beam.at(i)
+                if not stcls.c.is_final():
+                    self.moves.set_costs(beam.is_valid[i], beam.costs[i], stcls, gold)
+                    if follow_gold:
+                        for j in range(self.moves.n_moves):
+                            beam.is_valid[i][j] *= beam.costs[i][j] < 1
+        if follow_gold:
+            beam.advance(_transition_state, NULL, <void*>self.moves.c)
+        else:
+            beam.advance(_transition_state, _hash_state, <void*>self.moves.c)
+        beam.check_done(_check_final_state, NULL)
+
+
+# These are passed as callbacks to thinc.search.Beam
+cdef int _transition_state(void* _dest, void* _src, class_t clas, void* _moves) except -1:
+    dest = <StateClass>_dest
+    src = <StateClass>_src
+    moves = <const Transition*>_moves
+    dest.clone(src)
+    moves[clas].do(dest.c, moves[clas].label)
+
+
+cdef void* _init_state(Pool mem, int length, void* tokens) except NULL:
+    cdef StateClass st = StateClass.init(<const TokenC*>tokens, length)
+    ## Ensure sent_start is set to 0 throughout
+    #for i in range(st.c.length):
+    #    st.c._sent[i].sent_start = False
+    #    st.c._sent[i].l_edge = i
+    #    st.c._sent[i].r_edge = i
+    #st.fast_forward()
+    Py_INCREF(st)
+    return <void*>st
+
+
+cdef int _check_final_state(void* _state, void* extra_args) except -1:
+    return (<StateClass>_state).is_final()
+
+
+def _cleanup(Beam beam):
+    for i in range(beam.width):
+        Py_XDECREF(<PyObject*>beam._states[i].content)
+        Py_XDECREF(<PyObject*>beam._parents[i].content)
+
+
+cdef hash_t _hash_state(void* _state, void* _) except 0:
+    state = <StateClass>_state
+    if state.c.is_final():
+        return 1
+    else:
+        return state.c.hash()
+
+
+def _check_train_integrity(Beam pred, Beam gold, GoldParse gold_parse, TransitionSystem moves):
+    for i in range(pred.size):
+        if not pred._states[i].is_done or pred._states[i].loss == 0:
+            continue
+        state = <StateClass>pred.at(i)
+        if is_gold(state, gold_parse, moves.strings) == True:
+            for dep in gold_parse.orig_annot:
+                print(dep[1], dep[3], dep[4])
+            print("Cost", pred._states[i].loss)
+            for j in range(gold_parse.length):
+                print(gold_parse.orig_annot[j][1], state.H(j), moves.strings[state.safe_get(j).dep])
+            acts = [moves.c[clas].move for clas in pred.histories[i]]
+            labels = [moves.c[clas].label for clas in pred.histories[i]]
+            print([moves.move_name(move, label) for move, label in zip(acts, labels)])
+            raise Exception("Predicted state is gold-standard")
+    for i in range(gold.size):
+        if not gold._states[i].is_done:
+            continue
+        state = <StateClass>gold.at(i)
+        if is_gold(state, gold_parse, moves.strings) == False:
+            print("Truth")
+            for dep in gold_parse.orig_annot:
+                print(dep[1], dep[3], dep[4])
+            print("Predicted good")
+            for j in range(gold_parse.length):
+                print(gold_parse.orig_annot[j][1], state.H(j), moves.strings[state.safe_get(j).dep])
+            raise Exception("Gold parse is not gold-standard")
+
+
+def is_gold(StateClass state, GoldParse gold, StringStore strings):
+    predicted = set()
+    truth = set()
+    for i in range(gold.length):
+        if state.safe_get(i).dep:
+            predicted.add((i, state.H(i), strings[state.safe_get(i).dep]))
+        else:
+            predicted.add((i, state.H(i), 'ROOT'))
+        id_, word, tag, head, dep, ner = gold.orig_annot[i]
+        truth.add((id_, head, dep))
+    return truth == predicted
+

spacy/syntax/iterators.pyx

@@ -10,7 +10,7 @@ def english_noun_chunks(doc):
     for i, word in enumerate(doc):
         if word.pos in (NOUN, PROPN, PRON) and word.dep in np_deps:
             yield word.left_edge.i, word.i+1, np_label
-        elif word.pos == NOUN and word.dep == conj:
+        elif word.pos in (NOUN, PROPN, PRON) and word.dep == conj:
             head = word.head
             while head.dep == conj and head.head.i < head.i:
                 head = head.head

spacy/syntax/parser.pxd

@@ -1,20 +1,14 @@
-from thinc.linear.avgtron cimport AveragedPerceptron
-from thinc.extra.eg cimport Example
-from thinc.structs cimport ExampleC
-
-from .stateclass cimport StateClass
 from .arc_eager cimport TransitionSystem
-from ..tokens.doc cimport Doc
-from ..structs cimport TokenC
 from ._state cimport StateC
+from ..structs cimport TokenC
 
+from thinc.base cimport Model
+from thinc.linalg cimport *
 
-cdef class ParserModel(AveragedPerceptron):
-    cdef void set_featuresC(self, ExampleC* eg, const StateC* state) nogil
 
 cdef class Parser:
-    cdef readonly ParserModel model
+    cdef readonly Model model
     cdef readonly TransitionSystem moves
     cdef int _projectivize
 
-    cdef int parseC(self, TokenC* tokens, int length, int nr_feat, int nr_class) nogil
+    cdef int parseC(self, TokenC* tokens, int length, int nr_feat, int nr_class) with gil

spacy/syntax/parser.pyx

@@ -1,4 +1,5 @@
 # cython: infer_types=True
+# cython: profile=True
 """
 MALT-style dependency parser
 """
@@ -12,19 +13,26 @@ from cpython.exc cimport PyErr_CheckSignals
 from libc.stdint cimport uint32_t, uint64_t
 from libc.string cimport memset, memcpy
 from libc.stdlib cimport malloc, calloc, free
+from libc.math cimport exp
 import os.path
 from os import path
 import shutil
 import json
 import sys
 from .nonproj import PseudoProjectivity
+import random
+import numpy.random
 
 from cymem.cymem cimport Pool, Address
 from murmurhash.mrmr cimport hash64
-from thinc.typedefs cimport weight_t, class_t, feat_t, atom_t, hash_t
+
+from thinc.typedefs cimport weight_t, class_t, feat_t, atom_t, hash_t, idx_t
 from thinc.linear.avgtron cimport AveragedPerceptron
 from thinc.linalg cimport VecVec
-from thinc.structs cimport SparseArrayC
+from thinc.structs cimport NeuralNetC, SparseArrayC, ExampleC
+from thinc.neural.nn cimport NeuralNet
+from thinc.extra.eg cimport Example
+
 from preshed.maps cimport MapStruct
 from preshed.maps cimport map_get
 from thinc.structs cimport FeatureC
@@ -44,8 +52,10 @@ from ..gold cimport GoldParse
 from . import _parse_features
 from ._parse_features cimport CONTEXT_SIZE
 from ._parse_features cimport fill_context
+from ._parse_features cimport *
 from .stateclass cimport StateClass
 from ._state cimport StateC
+from ._neural cimport ParserNeuralNet, ParserPerceptron
 
 
 DEBUG = False
@@ -60,8 +70,10 @@ def get_templates(name):
         return pf.ner
     elif name == 'debug':
         return pf.unigrams
-    elif name.startswith('embed'):
-        return (pf.words, pf.tags, pf.labels)
+    elif name.startswith('neural'):
+        features = pf.words + pf.tags + pf.labels
+        slots = [0] * len(pf.words) + [1] * len(pf.tags) + [2] * len(pf.labels)
+        return ([(f,) for f in features], slots)
     else:
         return (pf.unigrams + pf.s0_n0 + pf.s1_n0 + pf.s1_s0 + pf.s0_n1 + pf.n0_n1 + \
                 pf.tree_shape + pf.trigrams)
@@ -71,17 +83,10 @@ def ParserFactory(transition_system):
     return lambda strings, dir_: Parser(strings, dir_, transition_system)
 
 
-cdef class ParserModel(AveragedPerceptron):
-    cdef void set_featuresC(self, ExampleC* eg, const StateC* state) nogil: 
-        fill_context(eg.atoms, state)
-        eg.nr_feat = self.extracter.set_features(eg.features, eg.atoms)
-
-
 cdef class Parser:
-    def __init__(self, StringStore strings, transition_system, ParserModel model, int projectivize = 0):
+    def __init__(self, StringStore strings, transition_system, model, *args, **kwargs):
         self.moves = transition_system
         self.model = model
-        self._projectivize = projectivize
 
     @classmethod
     def from_dir(cls, model_dir, strings, transition_system):
@@ -91,12 +96,20 @@ cdef class Parser:
             print >> sys.stderr, "Warning: model path:", model_dir, "is not a directory"
         cfg = Config.read(model_dir, 'config')
         moves = transition_system(strings, cfg.labels)
-        templates = get_templates(cfg.features)
-        model = ParserModel(templates)
-        project = cfg.projectivize if hasattr(cfg,'projectivize') else False
+
+        if cfg.get('model') == 'neural':
+            model = ParserNeuralNet(cfg.hyper_params['hidden_layers'] + [moves.n_moves],
+                        update_step=cfg.hyper_params['update_step'],
+                        eta=cfg.hyper_params['learn_rate'],
+                        rho=cfg.hyper_params['L2'],
+                        noise=cfg.hyper_params['noise'])
+        else:
+            model = ParserPerceptron(get_templates(cfg.feat_set),
+                        learn_rate=cfg.get('eta', 0.001),
+                        l1_penalty=cfg.rho)
         if path.exists(path.join(model_dir, 'model')):
             model.load(path.join(model_dir, 'model'))
-        return cls(strings, moves, model, project)
+        return cls(strings, moves, model, beam_width=cfg.get('beam_width', 1))
 
     @classmethod
     def load(cls, pkg_or_str_or_file, vocab):
@@ -156,20 +169,14 @@ cdef class Parser:
             self.moves.finalize_doc(doc)
             yield doc
 
-    cdef int parseC(self, TokenC* tokens, int length, int nr_feat, int nr_class) nogil:
-        cdef ExampleC eg
-        eg.nr_feat = nr_feat
-        eg.nr_atom = CONTEXT_SIZE
-        eg.nr_class = nr_class
-        eg.features = <FeatureC*>calloc(sizeof(FeatureC), nr_feat)
-        eg.atoms = <atom_t*>calloc(sizeof(atom_t), CONTEXT_SIZE)
-        eg.scores = <weight_t*>calloc(sizeof(weight_t), nr_class)
-        eg.is_valid = <int*>calloc(sizeof(int), nr_class)
+    cdef int parseC(self, TokenC* tokens, int length, int nr_feat, int nr_class) with gil:
+        cdef Example py_eg = Example(nr_class=nr_class, nr_feat=nr_feat)
+        cdef ExampleC* eg = py_eg.c
         state = new StateC(tokens, length)
         self.moves.initialize_state(state)
         cdef int i
         while not state.is_final():
-            self.model.set_featuresC(&eg, state)
+            eg.nr_feat = self.model.set_featuresC(eg.features, state)
             self.moves.set_valid(eg.is_valid, state)
             self.model.set_scoresC(eg.scores, eg.features, eg.nr_feat)
 
@@ -177,49 +184,37 @@ cdef class Parser:
 
             action = self.moves.c[guess]
             if not eg.is_valid[guess]:
-                # with gil:
-                #     move_name = self.moves.move_name(action.move, action.label)
-                #     print 'invalid action:', move_name
                 return 1
 
             action.do(state, action.label)
-            memset(eg.scores, 0, sizeof(eg.scores[0]) * eg.nr_class)
-            for i in range(eg.nr_class):
-                eg.is_valid[i] = 1
+            py_eg.reset()
         self.moves.finalize_state(state)
         for i in range(length):
             tokens[i] = state._sent[i]
         del state
-        free(eg.features)
-        free(eg.atoms)
-        free(eg.scores)
-        free(eg.is_valid)
         return 0
   
-    def train(self, Doc tokens, GoldParse gold):
+    def train(self, Doc tokens, GoldParse gold, itn=0):
         self.moves.preprocess_gold(gold)
         cdef StateClass stcls = StateClass.init(tokens.c, tokens.length)
         self.moves.initialize_state(stcls.c)
-        cdef Pool mem = Pool()
         cdef Example eg = Example(
                 nr_class=self.moves.n_moves,
-                nr_atom=CONTEXT_SIZE,
                 nr_feat=self.model.nr_feat)
-        cdef weight_t loss = 0
+        loss = 0
         cdef Transition action
         while not stcls.is_final():
-            self.model.set_featuresC(&eg.c, stcls.c)
-            self.moves.set_costs(eg.c.is_valid, eg.c.costs, stcls, gold)
+            eg.c.nr_feat = self.model.set_featuresC(eg.c.features, stcls.c)
+            self.model.dropoutC(eg.c.features,
+                0.5, eg.c.nr_feat)
+            if eg.c.features[0].key == 1:
+                eg.c.features[0].value = 1.0
             self.model.set_scoresC(eg.c.scores, eg.c.features, eg.c.nr_feat)
-            self.model.updateC(&eg.c)
-            guess = VecVec.arg_max_if_true(eg.c.scores, eg.c.is_valid, eg.c.nr_class)
-
+            self.moves.set_costs(eg.c.is_valid, eg.c.costs, stcls, gold)
             action = self.moves.c[eg.guess]
             action.do(stcls.c, action.label)
-            loss += eg.costs[eg.guess]
-            eg.fill_scores(0, eg.nr_class)
-            eg.fill_costs(0, eg.nr_class)
-            eg.fill_is_valid(0, eg.nr_class)
+            loss += self.model.update(eg)
+            eg.reset()
         return loss
 
     def step_through(self, Doc doc):
@@ -280,7 +275,7 @@ cdef class StepwiseState:
 
     def predict(self):
         self.eg.reset()
-        self.parser.model.set_featuresC(&self.eg.c, self.stcls.c)
+        self.eg.c.nr_feat = self.parser.model.set_featuresC(self.eg.c.features, self.stcls.c)
         self.parser.moves.set_valid(self.eg.c.is_valid, self.stcls.c)
         self.parser.model.set_scoresC(self.eg.c.scores,
             self.eg.c.features, self.eg.c.nr_feat)

spacy/tagger.pxd

@@ -1,13 +1,13 @@
 from thinc.linear.avgtron cimport AveragedPerceptron
 from thinc.extra.eg cimport Example
-from thinc.structs cimport ExampleC
+from thinc.structs cimport ExampleC, FeatureC
 
 from .structs cimport TokenC
 from .vocab cimport Vocab
 
 
 cdef class TaggerModel(AveragedPerceptron):
-    cdef void set_featuresC(self, ExampleC* eg, const TokenC* tokens, int i) except *
+    cdef int set_featuresC(self, FeatureC* feats, const void* _token) nogil
  
 
 cdef class Tagger:

spacy/tagger.pyx

@@ -71,15 +71,17 @@ cpdef enum:
 
 
 cdef class TaggerModel(AveragedPerceptron):
-    cdef void set_featuresC(self, ExampleC* eg, const TokenC* tokens, int i) except *:
-        
-        _fill_from_token(&eg.atoms[P2_orth], &tokens[i-2])
-        _fill_from_token(&eg.atoms[P1_orth], &tokens[i-1])
-        _fill_from_token(&eg.atoms[W_orth], &tokens[i])
-        _fill_from_token(&eg.atoms[N1_orth], &tokens[i+1])
-        _fill_from_token(&eg.atoms[N2_orth], &tokens[i+2])
+    cdef int set_featuresC(self, FeatureC* features, const void* _token) nogil:
+        cdef atom_t[N_CONTEXT_FIELDS] context
+        memset(context, 0, sizeof(context))
+        token = <const TokenC*>_token
+        _fill_from_token(&context[P2_orth], token - 2)
+        _fill_from_token(&context[P1_orth], token - 1)
+        _fill_from_token(&context[W_orth], token)
+        _fill_from_token(&context[N1_orth], token + 1)
+        _fill_from_token(&context[N2_orth], token + 2)
 
-        eg.nr_feat = self.extracter.set_features(eg.features, eg.atoms)
+        return self.extracter.set_features(features, context)
 
 
 cdef inline void _fill_from_token(atom_t* context, const TokenC* t) nogil:
@@ -153,7 +155,7 @@ cdef class Tagger:
     @classmethod
     def from_package(cls, pkg, vocab):
         # TODO: templates.json deprecated? not present in latest package
-        # templates = cls.default_templates()
+        #templates = cls.default_templates()
         templates = pkg.load_json(('pos', 'templates.json'), default=cls.default_templates())
 
         model = TaggerModel(templates)
@@ -202,12 +204,13 @@ cdef class Tagger:
                                   nr_feat=self.model.nr_feat)
         for i in range(tokens.length):
             if tokens.c[i].pos == 0:                
-                self.model.set_featuresC(&eg.c, tokens.c, i)
+                eg.c.nr_feat = self.model.set_featuresC(eg.c.features, &tokens.c[i])
                 self.model.set_scoresC(eg.c.scores,
                     eg.c.features, eg.c.nr_feat)
                 guess = VecVec.arg_max_if_true(eg.c.scores, eg.c.is_valid, eg.c.nr_class)
                 self.vocab.morphology.assign_tag(&tokens.c[i], guess)
                 eg.fill_scores(0, eg.c.nr_class)
+                eg.reset()
         tokens.is_tagged = True
         tokens._py_tokens = [None] * tokens.length
 
@@ -231,18 +234,15 @@ cdef class Tagger:
             nr_class=self.vocab.morphology.n_tags,
             nr_feat=self.model.nr_feat)
         for i in range(tokens.length):
-            self.model.set_featuresC(&eg.c, tokens.c, i)
+            eg.c.nr_feat = self.model.set_featuresC(eg.c.features, &tokens.c[i])
             eg.costs = [ 1 if golds[i] not in (c, -1) else 0 for c in xrange(eg.nr_class) ]
             self.model.set_scoresC(eg.c.scores,
                 eg.c.features, eg.c.nr_feat)
-            self.model.updateC(&eg.c)
-
             self.vocab.morphology.assign_tag(&tokens.c[i], eg.guess)
-            
+            self.model.update(eg)
             correct += eg.cost == 0
             self.freqs[TAG][tokens.c[i].tag] += 1
-            eg.fill_scores(0, eg.c.nr_class)
-            eg.fill_costs(0, eg.c.nr_class)
+            eg.reset()
         tokens.is_tagged = True
         tokens._py_tokens = [None] * tokens.length
         return correct