Merge master

examples/training/conllu.py

@@ -18,6 +18,7 @@ from spacy.syntax.nonproj import projectivize
 from collections import defaultdict, Counter
 from timeit import default_timer as timer
 from spacy.matcher import Matcher
+from spacy.morphology import Fused_begin, Fused_inside
 
 import itertools
 import random
@@ -86,18 +87,28 @@ def read_data(nlp, conllu_file, text_file, raw_text=True, oracle_segments=False,
         sent_annots = []
         for cs in cd:
             sent = defaultdict(list)
+            fused_ids = set()
             for id_, word, lemma, pos, tag, morph, head, dep, _, space_after in cs:
                 if '.' in id_:
                     continue
                 if '-' in id_:
+                    fuse_start, fuse_end = id_.split('-')
+                    for sub_id in range(int(fuse_start), int(fuse_end)+1):
+                        fused_ids.add(str(sub_id))
+                    sent['tokens'].append(word)
                     continue
+                if id_ not in fused_ids:
+                    sent['tokens'].append(word)
+                    if space_after == '_':
+                        sent['tokens'][-1] += ' '
+                elif id_ == fuse_end and space_after == '_':
+                    sent['tokens'][-1] += ' '
                 id_ = int(id_)-1
                 head = int(head)-1 if head != '0' else id_
                 sent['words'].append(word)
                 sent['tags'].append(tag)
                 sent['heads'].append(head)
                 sent['deps'].append('ROOT' if dep == 'root' else dep)
-                sent['spaces'].append(space_after == '_')
             sent['entities'] = ['-'] * len(sent['words'])
             sent['heads'], sent['deps'] = projectivize(sent['heads'],
                                                        sent['deps'])
@@ -155,14 +166,13 @@ def _make_gold(nlp, text, sent_annots):
     flat = defaultdict(list)
     for sent in sent_annots:
         flat['heads'].extend(len(flat['words'])+head for head in sent['heads'])
-        for field in ['words', 'tags', 'deps', 'entities', 'spaces']:
+        for field in ['words', 'tags', 'deps', 'entities', 'tokens']:
             flat[field].extend(sent[field])
     # Construct text if necessary
-    assert len(flat['words']) == len(flat['spaces'])
     if text is None:
-        text = ''.join(word+' '*space for word, space in zip(flat['words'], flat['spaces'])) 
+        text = ''.join(flat['tokens'])
     doc = nlp.make_doc(text)
-    flat.pop('spaces')
+    flat.pop('tokens')
     gold = GoldParse(doc, **flat)
     return doc, gold
 
@@ -212,12 +222,42 @@ def write_conllu(docs, file_):
         file_.write("# newdoc id = {i}\n".format(i=i))
         for j, sent in enumerate(doc.sents):
             file_.write("# sent_id = {i}.{j}\n".format(i=i, j=j))
-            file_.write("# text = {text}\n".format(text=sent.text))
+            file_.write('# text = {text}\n'.format(text=sent.text))
             for k, token in enumerate(sent):
-                file_.write(token._.get_conllu_lines(k) + '\n')
+                file_.write(_get_token_conllu(token, k, len(sent)) + '\n')
             file_.write('\n')
 
 
+def _get_token_conllu(token, k, sent_len):
+    if token.check_morph(Fused_begin) and (k+1 < sent_len):
+        n = 1
+        text = [token.text]
+        while token.nbor(n).check_morph(Fused_inside):
+            text.append(token.nbor(n).text)
+            n += 1
+        id_ = '%d-%d' % (k+1, (k+n))
+        fields = [id_, ''.join(text)] + ['_'] * 8
+        lines = ['\t'.join(fields)]
+    else:
+        lines = []
+    if token.head.i == token.i:
+        head = 0
+    else:
+        head = k + (token.head.i - token.i) + 1
+    fields = [str(k+1), token.text, token.lemma_, token.pos_, token.tag_, '_',
+              str(head), token.dep_.lower(), '_', '_']
+    if token.check_morph(Fused_begin) and (k+1 < sent_len):
+        if k == 0:
+            fields[1] = token.norm_[0].upper() + token.norm_[1:]
+        else:
+            fields[1] = token.norm_
+    elif token.check_morph(Fused_inside):
+        fields[1] = token.norm_
+
+    lines.append('\t'.join(fields))
+    return '\n'.join(lines)
+
+
 def print_progress(itn, losses, ud_scores):
     fields = {
         'dep_loss': losses.get('parser', 0.0),
@@ -267,7 +307,6 @@ Token.set_extension('get_conllu_lines', method=get_token_conllu)
 Token.set_extension('begins_fused', default=False)
 Token.set_extension('inside_fused', default=False)
 
-
 ##################
 # Initialization #
 ##################
@@ -280,14 +319,63 @@ def load_nlp(corpus, config):
         nlp.vocab.from_disk(config.vectors / 'vocab')
     return nlp
 
+def extract_tokenizer_exceptions(paths):
+    with paths.train.conllu.open() as file_:
+        conllu = read_conllu(file_)
+    fused = defaultdict(lambda: defaultdict(list))
+    for doc in conllu:
+        for sent in doc:
+            for i, token in enumerate(sent):
+                if '-' in token[0]:
+                    start, end = token[0].split('-')
+                    length = int(end) - int(start)
+                    subtokens = sent[i+1 : i+1+length+1]
+                    forms = [t[1].lower() for t in subtokens]
+                    fused[token[1]][tuple(forms)].append(subtokens)
+    exc = {}
+    for word, expansions in fused.items():
+        by_freq = [(len(occurs), key, occurs) for key, occurs in expansions.items()]
+        freq, key, occurs = max(by_freq)
+        if word == ''.join(key):
+            # Happy case: we get a perfect split, with each letter accounted for.
+            analysis = [{'ORTH': subtoken} for subtoken in key]
+        elif len(word) == sum(len(subtoken) for subtoken in key):
+            # Unideal, but at least lengths match.
+            analysis = []
+            remain = word
+            for subtoken in key:
+                analysis.append({'ORTH': remain[:len(subtoken)]})
+                remain = remain[len(subtoken):]
+            assert len(remain) == 0, (word, key, remain)
+        else:
+            # Let's say word is 6 long, and there are three subtokens. The orths
+            # *must* equal the original string. Arbitrarily, split [4, 1, 1]
+            first = word[:len(word)-(len(key)-1)]
+            subtokens = [first]
+            remain = word[len(first):]
+            for i in range(1, len(key)):
+                subtokens.append(remain[:1])
+                remain = remain[1:]
+            assert len(remain) == 0, (word, subtokens, remain)
+            analysis = [{'ORTH': subtoken} for subtoken in subtokens]
+            for i, token in enumerate(occurs[0]):
+                analysis[i]['NORM'] = token[1]
+        analysis[0]['morphology'] = [Fused_begin]
+        for subtoken in analysis[1:]:
+            subtoken['morphology'] = [Fused_inside]
+        exc[word] = analysis
+    return exc
+
 def initialize_pipeline(nlp, docs, golds, config):
+    nlp.add_pipe(nlp.create_pipe('tagger'))
     nlp.add_pipe(nlp.create_pipe('parser'))
+    nlp.parser.moves.add_action(2, 'subtok')
     if config.multitask_tag:
         nlp.parser.add_multitask_objective('tag')
     if config.multitask_sent:
         nlp.parser.add_multitask_objective('sent_start')
-    nlp.parser.moves.add_action(2, 'subtok')
-    nlp.add_pipe(nlp.create_pipe('tagger'))
+    if config.multitask_dep:
+        nlp.parser.add_multitask_objective('dep')
     for gold in golds:
         for tag in gold.tags:
             if tag is not None:
@@ -310,6 +398,7 @@ def initialize_pipeline(nlp, docs, golds, config):
 class Config(object):
     vectors = attr.ib(default=None)
     max_doc_length = attr.ib(default=10)
+    multitask_dep = attr.ib(default=True)
     multitask_tag = attr.ib(default=True)
     multitask_sent = attr.ib(default=True)
     nr_epoch = attr.ib(default=30)
@@ -364,7 +453,9 @@ def main(ud_dir, parses_dir, config, corpus, limit=0):
         (parses_dir / corpus).mkdir()
     print("Train and evaluate", corpus, "using lang", paths.lang)
     nlp = load_nlp(paths.lang, config)
-
+    tokenizer_exceptions = extract_tokenizer_exceptions(paths)
+    for orth, subtokens in tokenizer_exceptions.items():
+        nlp.tokenizer.add_special_case(orth, subtokens)
     docs, golds = read_data(nlp, paths.train.conllu.open(), paths.train.text.open(),
                             max_doc_length=config.max_doc_length, limit=limit)
 

spacy/attrs.pyx

@@ -143,8 +143,10 @@ def intify_attrs(stringy_attrs, strings_map=None, _do_deprecated=False):
     for name, value in stringy_attrs.items():
         if isinstance(name, int):
             int_key = name
-        else:
+        elif name.upper() in IDS:
             int_key = IDS[name.upper()]
+        else:
+            continue
         if strings_map is not None and isinstance(value, basestring):
             if hasattr(strings_map, 'add'):
                 value = strings_map.add(value)

spacy/lang/de/tokenizer_exceptions.py

@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from ...symbols import ORTH, LEMMA, TAG, NORM, PRON_LEMMA
+from ...morphology import Fused_begin, Fused_inside
 
 
 _exc = {
@@ -47,7 +48,7 @@ _exc = {
 
     "über'm": [
         {ORTH: "über", LEMMA: "über"},
-        {ORTH: "'m", LEMMA: "der", NORM: "dem"}]
+        {ORTH: "'m", LEMMA: "der", NORM: "dem"}],
 }
 
 

spacy/morphology.pxd

@@ -31,6 +31,7 @@ cdef class Morphology:
     cdef public object reverse_index
     cdef public object tag_names
     cdef public object exc
+    cdef public object _morph2features
 
     cdef RichTagC* rich_tags
     cdef PreshMapArray _cache
@@ -42,6 +43,8 @@ cdef class Morphology:
     cdef int assign_tag_id(self, TokenC* token, int tag_id) except -1
 
     cdef int assign_feature(self, uint64_t* morph, univ_morph_t feat_id, bint value) except -1
+    
+    cdef int set_feature(self, uint64_t* morph, univ_morph_t feat_id, bint value) except -1
 
 
 cdef enum univ_morph_t:
@@ -298,4 +301,7 @@ cdef enum univ_morph_t:
     VerbType_mod # U
     VerbType_light # U
 
+    Fused_begin
+    Fused_inside
+
 

spacy/morphology.pyx

@@ -9,6 +9,7 @@ from .attrs import LEMMA, intify_attrs
 from .parts_of_speech cimport SPACE
 from .parts_of_speech import IDS as POS_IDS
 from .lexeme cimport Lexeme
+from .strings cimport hash_string
 
 
 def _normalize_props(props):
@@ -29,6 +30,11 @@ def _normalize_props(props):
             out[key] = value
     return out
 
+cdef uint64_t hash_features(features):
+    # TODO improve this
+    cdef unicode string = str(tuple(features))
+    return hash_string(string)
+
 
 cdef class Morphology:
     def __init__(self, StringStore string_store, tag_map, lemmatizer, exc=None):
@@ -36,7 +42,7 @@ cdef class Morphology:
         self.strings = string_store
         # Add special space symbol. We prefix with underscore, to make sure it
         # always sorts to the end.
-        space_attrs = tag_map.get('SP', {POS: SPACE})
+        space_attrs = tag_map.get('_SP', tag_map.get('SP', {POS: SPACE}))
         if '_SP' not in tag_map:
             self.strings.add('_SP')
             tag_map = dict(tag_map)
@@ -48,16 +54,19 @@ cdef class Morphology:
         self.reverse_index = {}
 
         self.rich_tags = <RichTagC*>self.mem.alloc(self.n_tags+1, sizeof(RichTagC))
+        self._morph2features = {}
         for i, (tag_str, attrs) in enumerate(sorted(tag_map.items())):
+            features = attrs.get('morphology', frozenset())
             self.strings.add(tag_str)
             self.tag_map[tag_str] = dict(attrs)
             attrs = _normalize_props(attrs)
             attrs = intify_attrs(attrs, self.strings, _do_deprecated=True)
             self.rich_tags[i].id = i
             self.rich_tags[i].name = self.strings.add(tag_str)
-            self.rich_tags[i].morph = 0
+            self.rich_tags[i].morph = hash_features(features)
             self.rich_tags[i].pos = attrs[POS]
             self.reverse_index[self.rich_tags[i].name] = i
+            self._morph2features[self.rich_tags[i].morph] = features
         # Add a 'null' tag, which we can reference when assign morphology to
         # untagged tokens.
         self.rich_tags[self.n_tags].id = self.n_tags
@@ -98,6 +107,7 @@ cdef class Morphology:
         # justification is that this is where the specific word and the tag
         # interact. Still, we should have a better way to enforce this rule, or
         # figure out why the statistical model fails. Related to Issue #220
+        previous_features = self.get_features(token.morph)
         if Lexeme.c_check_flag(token.lex, IS_SPACE):
             tag_id = self.reverse_index[self.strings.add('_SP')]
         rich_tag = self.rich_tags[tag_id]
@@ -113,13 +123,36 @@ cdef class Morphology:
         token.pos = analysis.tag.pos
         token.tag = analysis.tag.name
         token.morph = analysis.tag.morph
+        for feature in previous_features:
+            self.set_feature(&token.morph, feature, True)
 
-    cdef int assign_feature(self, uint64_t* flags, univ_morph_t flag_id, bint value) except -1:
-        cdef flags_t one = 1
-        if value:
-            flags[0] |= one << flag_id
+    cdef int assign_feature(self, uint64_t* morph, univ_morph_t flag_id, bint value) except -1:
+        # Deprecated
+        pass
+
+    cdef int set_feature(self, uint64_t* morph, univ_morph_t flag_id, bint value) except -1:
+        '''Update a morph attribute in-place, so that it indicates the given
+        feature.
+        '''
+        features = self._morph2features.get(morph[0], {})
+        cdef uint64_t key
+        cdef attr_t flag = flag_id
+        if (flag in features) != value:
+            new_features = set(features)
+            if value:
+                new_features.add(flag)
+            else:
+                new_features.remove(flag)
+            new_features = frozenset(new_features)
+            key = hash_features(new_features)
+            morph[0] = key
+            self._morph2features[morph[0]] = new_features
+
+    def get_features(self, uint64_t morph):
+        if morph in self._morph2features:
+            return self._morph2features[morph]
         else:
-            flags[0] &= ~(one << flag_id)
+            return frozenset()
 
     def add_special_case(self, unicode tag_str, unicode orth_str, attrs,
                          force=False):
@@ -140,6 +173,9 @@ cdef class Morphology:
         tag_id = self.reverse_index[tag]
         orth = self.strings[orth_str]
         cdef RichTagC rich_tag = self.rich_tags[tag_id]
+        features = attrs.get('morphology', frozenset())
+        cdef uint64_t morph = hash_features(features)
+        self._morph2features[morph] = features
         attrs = intify_attrs(attrs, self.strings, _do_deprecated=True)
         cached = <MorphAnalysisC*>self._cache.get(tag_id, orth)
         if cached is NULL:
@@ -152,12 +188,11 @@ cdef class Morphology:
                 "force=True to overwrite." % (tag_str, orth_str))
 
         cached.tag = rich_tag
+        cached.tag.morph = morph
         # TODO: Refactor this to take arbitrary attributes.
         for name_id, value_id in attrs.items():
             if name_id == LEMMA:
                 cached.lemma = value_id
-            else:
-                self.assign_feature(&cached.tag.morph, name_id, value_id)
         if cached.lemma == 0:
             cached.lemma = self.lemmatize(rich_tag.pos, orth, attrs)
         self._cache.set(tag_id, orth, <void*>cached)
@@ -318,122 +353,125 @@ IDS = {
     "AdvType_sta": AdvType_sta,
     "AdvType_ex": AdvType_ex,
     "AdvType_adadj": AdvType_adadj,
-    "ConjType_oper ": ConjType_oper,  # cz, U,
-    "ConjType_comp ": ConjType_comp,  # cz, U,
-    "Connegative_yes ": Connegative_yes,  # fi,
-    "Derivation_minen ": Derivation_minen,  # fi,
-    "Derivation_sti ": Derivation_sti,  # fi,
-    "Derivation_inen ": Derivation_inen,  # fi,
-    "Derivation_lainen ": Derivation_lainen,  # fi,
-    "Derivation_ja ": Derivation_ja,  # fi,
-    "Derivation_ton ": Derivation_ton,  # fi,
-    "Derivation_vs ": Derivation_vs,  # fi,
-    "Derivation_ttain ": Derivation_ttain,  # fi,
-    "Derivation_ttaa ": Derivation_ttaa,  # fi,
-    "Echo_rdp ": Echo_rdp,  # U,
-    "Echo_ech ": Echo_ech,  # U,
-    "Foreign_foreign ": Foreign_foreign,  # cz, fi, U,
-    "Foreign_fscript ": Foreign_fscript,  # cz, fi, U,
-    "Foreign_tscript ": Foreign_tscript,  # cz, U,
-    "Foreign_yes ": Foreign_yes,  # sl,
-    "Gender_dat_masc ": Gender_dat_masc,  # bq, U,
-    "Gender_dat_fem ": Gender_dat_fem,  # bq, U,
-    "Gender_erg_masc ": Gender_erg_masc,  # bq,
-    "Gender_erg_fem ": Gender_erg_fem,  # bq,
-    "Gender_psor_masc ": Gender_psor_masc,  # cz, sl, U,
-    "Gender_psor_fem ": Gender_psor_fem,  # cz, sl, U,
-    "Gender_psor_neut ": Gender_psor_neut,  # sl,
-    "Hyph_yes ": Hyph_yes,  # cz, U,
-    "InfForm_one ": InfForm_one,  # fi,
-    "InfForm_two ": InfForm_two,  # fi,
-    "InfForm_three ": InfForm_three,  # fi,
-    "NameType_geo ": NameType_geo,  # U, cz,
-    "NameType_prs ": NameType_prs,  # U, cz,
-    "NameType_giv ": NameType_giv,  # U, cz,
-    "NameType_sur ": NameType_sur,  # U, cz,
-    "NameType_nat ": NameType_nat,  # U, cz,
-    "NameType_com ": NameType_com,  # U, cz,
-    "NameType_pro ": NameType_pro,  # U, cz,
-    "NameType_oth ": NameType_oth,  # U, cz,
-    "NounType_com ": NounType_com,  # U,
-    "NounType_prop ": NounType_prop,  # U,
-    "NounType_class ": NounType_class,  # U,
-    "Number_abs_sing ": Number_abs_sing,  # bq, U,
-    "Number_abs_plur ": Number_abs_plur,  # bq, U,
-    "Number_dat_sing ": Number_dat_sing,  # bq, U,
-    "Number_dat_plur ": Number_dat_plur,  # bq, U,
-    "Number_erg_sing ": Number_erg_sing,  # bq, U,
-    "Number_erg_plur ": Number_erg_plur,  # bq, U,
-    "Number_psee_sing ": Number_psee_sing,  # U,
-    "Number_psee_plur ": Number_psee_plur,  # U,
-    "Number_psor_sing ": Number_psor_sing,  # cz, fi, sl, U,
-    "Number_psor_plur ": Number_psor_plur,  # cz, fi, sl, U,
-    "NumForm_digit ": NumForm_digit,  # cz, sl, U,
-    "NumForm_roman ": NumForm_roman,  # cz, sl, U,
-    "NumForm_word ": NumForm_word,  # cz, sl, U,
-    "NumValue_one ": NumValue_one,  # cz, U,
-    "NumValue_two ": NumValue_two,  # cz, U,
-    "NumValue_three ": NumValue_three,  # cz, U,
-    "PartForm_pres ": PartForm_pres,  # fi,
-    "PartForm_past ": PartForm_past,  # fi,
-    "PartForm_agt ": PartForm_agt,  # fi,
-    "PartForm_neg ": PartForm_neg,  # fi,
-    "PartType_mod ": PartType_mod,  # U,
-    "PartType_emp ": PartType_emp,  # U,
-    "PartType_res ": PartType_res,  # U,
-    "PartType_inf ": PartType_inf,  # U,
-    "PartType_vbp ": PartType_vbp,  # U,
-    "Person_abs_one ": Person_abs_one,  # bq, U,
-    "Person_abs_two ": Person_abs_two,  # bq, U,
-    "Person_abs_three ": Person_abs_three,  # bq, U,
-    "Person_dat_one ": Person_dat_one,  # bq, U,
-    "Person_dat_two ": Person_dat_two,  # bq, U,
-    "Person_dat_three ": Person_dat_three,  # bq, U,
-    "Person_erg_one ": Person_erg_one,  # bq, U,
-    "Person_erg_two ": Person_erg_two,  # bq, U,
-    "Person_erg_three ": Person_erg_three,  # bq, U,
-    "Person_psor_one ": Person_psor_one,  # fi, U,
-    "Person_psor_two ": Person_psor_two,  # fi, U,
-    "Person_psor_three ": Person_psor_three,  # fi, U,
-    "Polite_inf ": Polite_inf,  # bq, U,
-    "Polite_pol ": Polite_pol,  # bq, U,
-    "Polite_abs_inf ": Polite_abs_inf,  # bq, U,
-    "Polite_abs_pol ": Polite_abs_pol,  # bq, U,
-    "Polite_erg_inf ": Polite_erg_inf,  # bq, U,
-    "Polite_erg_pol ": Polite_erg_pol,  # bq, U,
-    "Polite_dat_inf ": Polite_dat_inf,  # bq, U,
-    "Polite_dat_pol ": Polite_dat_pol,  # bq, U,
-    "Prefix_yes ": Prefix_yes,  # U,
-    "PrepCase_npr ": PrepCase_npr,  # cz,
-    "PrepCase_pre ": PrepCase_pre,  # U,
-    "PunctSide_ini ": PunctSide_ini,  # U,
-    "PunctSide_fin ": PunctSide_fin,  # U,
-    "PunctType_peri ": PunctType_peri,  # U,
-    "PunctType_qest ": PunctType_qest,  # U,
-    "PunctType_excl ": PunctType_excl,  # U,
-    "PunctType_quot ": PunctType_quot,  # U,
-    "PunctType_brck ": PunctType_brck,  # U,
-    "PunctType_comm ": PunctType_comm,  # U,
-    "PunctType_colo ": PunctType_colo,  # U,
-    "PunctType_semi ": PunctType_semi,  # U,
-    "PunctType_dash ": PunctType_dash,  # U,
-    "Style_arch ": Style_arch,  # cz, fi, U,
-    "Style_rare ": Style_rare,  # cz, fi, U,
-    "Style_poet ": Style_poet,  # cz, U,
-    "Style_norm ": Style_norm,  # cz, U,
-    "Style_coll ": Style_coll,  # cz, U,
-    "Style_vrnc ": Style_vrnc,  # cz, U,
-    "Style_sing ": Style_sing,  # cz, U,
-    "Style_expr ": Style_expr,  # cz, U,
-    "Style_derg ": Style_derg,  # cz, U,
-    "Style_vulg ": Style_vulg,  # cz, U,
-    "Style_yes ": Style_yes,  # fi, U,
-    "StyleVariant_styleShort ": StyleVariant_styleShort,  # cz,
-    "StyleVariant_styleBound ": StyleVariant_styleBound,  # cz, sl,
-    "VerbType_aux ": VerbType_aux,  # U,
-    "VerbType_cop ": VerbType_cop,  # U,
-    "VerbType_mod ": VerbType_mod,  # U,
-    "VerbType_light ": VerbType_light,  # U,
+    "ConjType_oper": ConjType_oper,  # cz, U,
+    "ConjType_comp": ConjType_comp,  # cz, U,
+    "Connegative_yes": Connegative_yes,  # fi,
+    "Derivation_minen": Derivation_minen,  # fi,
+    "Derivation_sti": Derivation_sti,  # fi,
+    "Derivation_inen": Derivation_inen,  # fi,
+    "Derivation_lainen": Derivation_lainen,  # fi,
+    "Derivation_ja": Derivation_ja,  # fi,
+    "Derivation_ton": Derivation_ton,  # fi,
+    "Derivation_vs": Derivation_vs,  # fi,
+    "Derivation_ttain": Derivation_ttain,  # fi,
+    "Derivation_ttaa": Derivation_ttaa,  # fi,
+    "Echo_rdp": Echo_rdp,  # U,
+    "Echo_ech": Echo_ech,  # U,
+    "Foreign_foreign": Foreign_foreign,  # cz, fi, U,
+    "Foreign_fscript": Foreign_fscript,  # cz, fi, U,
+    "Foreign_tscript": Foreign_tscript,  # cz, U,
+    "Foreign_yes": Foreign_yes,  # sl,
+    "Gender_dat_masc": Gender_dat_masc,  # bq, U,
+    "Gender_dat_fem": Gender_dat_fem,  # bq, U,
+    "Gender_erg_masc": Gender_erg_masc,  # bq,
+    "Gender_erg_fem": Gender_erg_fem,  # bq,
+    "Gender_psor_masc": Gender_psor_masc,  # cz, sl, U,
+    "Gender_psor_fem": Gender_psor_fem,  # cz, sl, U,
+    "Gender_psor_neut": Gender_psor_neut,  # sl,
+    "Hyph_yes": Hyph_yes,  # cz, U,
+    "InfForm_one": InfForm_one,  # fi,
+    "InfForm_two": InfForm_two,  # fi,
+    "InfForm_three": InfForm_three,  # fi,
+    "NameType_geo": NameType_geo,  # U, cz,
+    "NameType_prs": NameType_prs,  # U, cz,
+    "NameType_giv": NameType_giv,  # U, cz,
+    "NameType_sur": NameType_sur,  # U, cz,
+    "NameType_nat": NameType_nat,  # U, cz,
+    "NameType_com": NameType_com,  # U, cz,
+    "NameType_pro": NameType_pro,  # U, cz,
+    "NameType_oth": NameType_oth,  # U, cz,
+    "NounType_com": NounType_com,  # U,
+    "NounType_prop": NounType_prop,  # U,
+    "NounType_class": NounType_class,  # U,
+    "Number_abs_sing": Number_abs_sing,  # bq, U,
+    "Number_abs_plur": Number_abs_plur,  # bq, U,
+    "Number_dat_sing": Number_dat_sing,  # bq, U,
+    "Number_dat_plur": Number_dat_plur,  # bq, U,
+    "Number_erg_sing": Number_erg_sing,  # bq, U,
+    "Number_erg_plur": Number_erg_plur,  # bq, U,
+    "Number_psee_sing": Number_psee_sing,  # U,
+    "Number_psee_plur": Number_psee_plur,  # U,
+    "Number_psor_sing": Number_psor_sing,  # cz, fi, sl, U,
+    "Number_psor_plur": Number_psor_plur,  # cz, fi, sl, U,
+    "NumForm_digit": NumForm_digit,  # cz, sl, U,
+    "NumForm_roman": NumForm_roman,  # cz, sl, U,
+    "NumForm_word": NumForm_word,  # cz, sl, U,
+    "NumValue_one": NumValue_one,  # cz, U,
+    "NumValue_two": NumValue_two,  # cz, U,
+    "NumValue_three": NumValue_three,  # cz, U,
+    "PartForm_pres": PartForm_pres,  # fi,
+    "PartForm_past": PartForm_past,  # fi,
+    "PartForm_agt": PartForm_agt,  # fi,
+    "PartForm_neg": PartForm_neg,  # fi,
+    "PartType_mod": PartType_mod,  # U,
+    "PartType_emp": PartType_emp,  # U,
+    "PartType_res": PartType_res,  # U,
+    "PartType_inf": PartType_inf,  # U,
+    "PartType_vbp": PartType_vbp,  # U,
+    "Person_abs_one": Person_abs_one,  # bq, U,
+    "Person_abs_two": Person_abs_two,  # bq, U,
+    "Person_abs_three": Person_abs_three,  # bq, U,
+    "Person_dat_one": Person_dat_one,  # bq, U,
+    "Person_dat_two": Person_dat_two,  # bq, U,
+    "Person_dat_three": Person_dat_three,  # bq, U,
+    "Person_erg_one": Person_erg_one,  # bq, U,
+    "Person_erg_two": Person_erg_two,  # bq, U,
+    "Person_erg_three": Person_erg_three,  # bq, U,
+    "Person_psor_one": Person_psor_one,  # fi, U,
+    "Person_psor_two": Person_psor_two,  # fi, U,
+    "Person_psor_three": Person_psor_three,  # fi, U,
+    "Polite_inf": Polite_inf,  # bq, U,
+    "Polite_pol": Polite_pol,  # bq, U,
+    "Polite_abs_inf": Polite_abs_inf,  # bq, U,
+    "Polite_abs_pol": Polite_abs_pol,  # bq, U,
+    "Polite_erg_inf": Polite_erg_inf,  # bq, U,
+    "Polite_erg_pol": Polite_erg_pol,  # bq, U,
+    "Polite_dat_inf": Polite_dat_inf,  # bq, U,
+    "Polite_dat_pol": Polite_dat_pol,  # bq, U,
+    "Prefix_yes": Prefix_yes,  # U,
+    "PrepCase_npr": PrepCase_npr,  # cz,
+    "PrepCase_pre": PrepCase_pre,  # U,
+    "PunctSide_ini": PunctSide_ini,  # U,
+    "PunctSide_fin": PunctSide_fin,  # U,
+    "PunctType_peri": PunctType_peri,  # U,
+    "PunctType_qest": PunctType_qest,  # U,
+    "PunctType_excl": PunctType_excl,  # U,
+    "PunctType_quot": PunctType_quot,  # U,
+    "PunctType_brck": PunctType_brck,  # U,
+    "PunctType_comm": PunctType_comm,  # U,
+    "PunctType_colo": PunctType_colo,  # U,
+    "PunctType_semi": PunctType_semi,  # U,
+    "PunctType_dash": PunctType_dash,  # U,
+    "Style_arch": Style_arch,  # cz, fi, U,
+    "Style_rare": Style_rare,  # cz, fi, U,
+    "Style_poet": Style_poet,  # cz, U,
+    "Style_norm": Style_norm,  # cz, U,
+    "Style_coll": Style_coll,  # cz, U,
+    "Style_vrnc": Style_vrnc,  # cz, U,
+    "Style_sing": Style_sing,  # cz, U,
+    "Style_expr": Style_expr,  # cz, U,
+    "Style_derg": Style_derg,  # cz, U,
+    "Style_vulg": Style_vulg,  # cz, U,
+    "Style_yes": Style_yes,  # fi, U,
+    "StyleVariant_styleShort": StyleVariant_styleShort,  # cz,
+    "StyleVariant_styleBound": StyleVariant_styleBound,  # cz, sl,
+    "VerbType_aux": VerbType_aux,  # U,
+    "VerbType_cop": VerbType_cop,  # U,
+    "VerbType_mod": VerbType_mod,  # U,
+    "VerbType_light": VerbType_light,  # U,
+
+    "Fused_begin": Fused_begin,   # Internal
+    "Fused_inside": Fused_inside # Internal
 }
 
 

spacy/pipeline.pyx

@@ -484,9 +484,11 @@ class Tagger(Pipe):
                         new_tag_map[tag] = {POS: X}
         cdef Vocab vocab = self.vocab
         if new_tag_map:
+            morph_feats = self.vocab.morphology._morph2features
             vocab.morphology = Morphology(vocab.strings, new_tag_map,
                                           vocab.morphology.lemmatizer,
                                           exc=vocab.morphology.exc)
+            vocab.morphology._morph2features = morph_feats
         if self.model is True:
             self.cfg['pretrained_dims'] = self.vocab.vectors.data.shape[1]
             self.model = self.Model(self.vocab.morphology.n_tags, **self.cfg)
@@ -519,10 +521,12 @@ class Tagger(Pipe):
         if values is None:
             values = {POS: "X"}
         tag_map[label] = values
+        morph_feats = self.vocab.morphology._morph2features
         self.vocab.morphology = Morphology(
             self.vocab.strings, tag_map=tag_map,
             lemmatizer=self.vocab.morphology.lemmatizer,
             exc=self.vocab.morphology.exc)
+        self.vocab.morphology._morph2features = morph_feats
         return 1
 
     def use_params(self, params):
@@ -554,10 +558,12 @@ class Tagger(Pipe):
 
         def load_tag_map(b):
             tag_map = msgpack.loads(b, encoding='utf8')
+            morph_feats = self.vocab.morphology._morph2features
             self.vocab.morphology = Morphology(
                 self.vocab.strings, tag_map=tag_map,
                 lemmatizer=self.vocab.morphology.lemmatizer,
                 exc=self.vocab.morphology.exc)
+            self.vocab.morphology._morph2features = morph_feats
 
         deserialize = OrderedDict((
             ('vocab', lambda b: self.vocab.from_bytes(b)),
@@ -590,10 +596,12 @@ class Tagger(Pipe):
         def load_tag_map(p):
             with p.open('rb') as file_:
                 tag_map = msgpack.loads(file_.read(), encoding='utf8')
+            morph_feats = self.vocab.morphology._morph2features
             self.vocab.morphology = Morphology(
                 self.vocab.strings, tag_map=tag_map,
                 lemmatizer=self.vocab.morphology.lemmatizer,
                 exc=self.vocab.morphology.exc)
+            self.vocab.morphology._morph2features = morph_feats
 
         deserialize = OrderedDict((
             ('cfg', lambda p: self.cfg.update(_load_cfg(p))),

spacy/symbols.pxd

@@ -384,6 +384,9 @@ cdef enum symbol_t:
     VerbType_cop # U
     VerbType_mod # U
     VerbType_light # U
+    
+    Fused_begin
+    Fused_inside
 
     PERSON
     NORP

spacy/symbols.pyx

@@ -389,6 +389,9 @@ IDS = {
     "VerbType_cop": VerbType_cop, # U,
     "VerbType_mod": VerbType_mod, # U,
     "VerbType_light": VerbType_light, # U,
+    
+    "Fused_begin": Fused_begin,
+    "Fused_inside": Fused_inside,
 
     "PERSON": PERSON,
     "NORP": NORP,

spacy/tests/lang/de/test_text.py

@@ -5,6 +5,7 @@
 from __future__ import unicode_literals
 
 import pytest
+from ....morphology import Fused_begin, Fused_inside
 
 
 def test_tokenizer_handles_long_text(de_tokenizer):
@@ -22,9 +23,15 @@ Umfang kläglich dünnen Beine flimmerten ihm hilflos vor den Augen.
 »Was ist mit mir geschehen?«, dachte er."""
 
     tokens = de_tokenizer(text)
-    assert len(tokens) == 109
+    assert len(tokens) == 110
 
 
+def test_fused(de_tokenizer):
+    doc = de_tokenizer('zum')
+    assert len(doc) == 2
+    assert doc[0].check_morph(Fused_begin)
+    assert doc[1].check_morph(Fused_inside)
+
 @pytest.mark.parametrize('text', [
     "Donaudampfschifffahrtsgesellschaftskapitänsanwärterposten",
     "Rindfleischetikettierungsüberwachungsaufgabenübertragungsgesetz",

spacy/tokens/token.pyx

@@ -10,6 +10,7 @@ cimport numpy as np
 np.import_array()
 import numpy
 
+from ..morphology cimport univ_morph_t
 from ..typedefs cimport hash_t
 from ..lexeme cimport Lexeme
 from .. import parts_of_speech
@@ -128,6 +129,15 @@ cdef class Token:
         """
         return Lexeme.c_check_flag(self.c.lex, flag_id)
 
+    def set_morph(self, univ_morph_t feature, bint value):
+        '''Set a morphological feature'''
+        self.vocab.morphology.set_feature(&self.c.morph, feature, value)
+
+    def check_morph(self, univ_morph_t feature):
+        '''Check whether the token has the given morphological feature.'''
+        features = self.vocab.morphology.get_features(self.c.morph)
+        return feature in features
+
     def nbor(self, int i=1):
         """Get a neighboring token.
 

spacy/vocab.pyx

@@ -232,14 +232,17 @@ cdef class Vocab:
         cdef int i
         tokens = <TokenC*>self.mem.alloc(len(substrings) + 1, sizeof(TokenC))
         for i, props in enumerate(substrings):
+            features = props.get('morphology', frozenset())
             props = intify_attrs(props, strings_map=self.strings,
-                                 _do_deprecated=True)
+                                 _do_deprecated=False)
             token = &tokens[i]
             # Set the special tokens up to have arbitrary attributes
             lex = <LexemeC*>self.get_by_orth(self.mem, props[ORTH])
             token.lex = lex
             if TAG in props:
                 self.morphology.assign_tag(token, props[TAG])
+            for feature in features:
+                self.morphology.set_feature(&token.morph, feature, True)
             for attr_id, value in props.items():
                 Token.set_struct_attr(token, attr_id, value)
                 Lexeme.set_struct_attr(lex, attr_id, value)