* Require Jieba Chinese tokenizer and tagger

bin/init_model.py

@@ -36,11 +36,8 @@ from spacy.strings import hash_string
 from preshed.counter import PreshCounter
 
 from spacy.parts_of_speech import NOUN, VERB, ADJ
+from spacy.util import get_lang_class
 
-import spacy.en
-import spacy.de
-import spacy.fi
-import spacy.it
 
 try:
     unicode
@@ -197,13 +194,6 @@ def setup_vocab(get_lex_attr, tag_map, src_dir, dst_dir):
 
 
 def main(lang_id, lang_data_dir, corpora_dir, model_dir):
-    languages = {
-        'en': spacy.en.English.default_lex_attrs(),
-        'de': spacy.de.German.default_lex_attrs(),
-        'fi': spacy.fi.Finnish.default_lex_attrs(),
-        'it': spacy.it.Italian.default_lex_attrs(),
-    }
-
     model_dir = Path(model_dir)
     lang_data_dir = Path(lang_data_dir) / lang_id
     corpora_dir = Path(corpora_dir) / lang_id
@@ -216,7 +206,8 @@ def main(lang_id, lang_data_dir, corpora_dir, model_dir):
 
     tag_map = json.load((lang_data_dir / 'tag_map.json').open())
     setup_tokenizer(lang_data_dir, model_dir / 'tokenizer')
-    setup_vocab(languages[lang_id], tag_map, corpora_dir, model_dir / 'vocab')
+    setup_vocab(get_lang_class(lang_id).default_lex_attrs(), tag_map, corpora_dir,
+                model_dir / 'vocab')
 
     if (lang_data_dir / 'gazetteer.json').exists():
         copyfile(str(lang_data_dir / 'gazetteer.json'),

bin/parser/train.py

@@ -13,8 +13,6 @@ import plac
 import re
 
 import spacy.util
-from spacy.en import English
-from spacy.de import German
 
 from spacy.syntax.util import Config
 from spacy.gold import read_json_file
@@ -43,17 +41,21 @@ def _corrupt(c, noise_level):
 
 
 def add_noise(orig, noise_level):
-    if random.random() >= noise_level:
-        return orig
-    elif type(orig) == list:
-        corrupted = [_corrupt(word, noise_level) for word in orig]
-        corrupted = [w for w in corrupted if w]
-        return corrupted
-    else:
-        return ''.join(_corrupt(c, noise_level) for c in orig)
+    # TODO
+    return orig.replace(' ', '')
+    #if random.random() >= noise_level:
+    #    return orig
+    #elif type(orig) == list:
+    #    corrupted = [_corrupt(word, noise_level) for word in orig]
+    #    corrupted = [w for w in corrupted if w]
+    #    return corrupted
+    #else:
+    #    return ''.join(_corrupt(c, noise_level) for c in orig)
 
 
 def score_model(scorer, nlp, raw_text, annot_tuples, verbose=False):
+    # TODO
+    raw_text = raw_text.replace(' ', '')
     if raw_text is None:
         tokens = nlp.tokenizer.tokens_from_list(annot_tuples[1])
     else:
@@ -62,7 +64,7 @@ def score_model(scorer, nlp, raw_text, annot_tuples, verbose=False):
     nlp.entity(tokens)
     nlp.parser(tokens)
     gold = GoldParse(tokens, annot_tuples)
-    scorer.score(tokens, gold, verbose=verbose)
+    scorer.score(tokens, gold, verbose=verbose, punct_labels=('p', 'punct', 'char'))
 
 
 def _merge_sents(sents):
@@ -141,7 +143,10 @@ def train(Language, gold_tuples, model_dir, n_iter=15, feat_set=u'basic',
                 nlp.tagger(tokens)
                 gold = GoldParse(tokens, annot_tuples)
                 if not gold.is_projective:
-                    raise Exception("Non-projective sentence in training: %s" % annot_tuples[1])
+                    # TODO
+                    nlp.tagger.train(tokens, gold.tags)
+                    continue
+                    #raise Exception("Non-projective sentence in training: %s" % annot_tuples[1])
                 loss += nlp.parser.train(tokens, gold)
                 nlp.entity.train(tokens, gold)
                 nlp.tagger.train(tokens, gold.tags)
@@ -176,7 +181,7 @@ def evaluate(Language, gold_tuples, model_dir, gold_preproc=False, verbose=False
             else:
                 tokens = nlp(raw_text)
             gold = GoldParse(tokens, annot_tuples)
-            scorer.score(tokens, gold, verbose=verbose)
+            scorer.score(tokens, gold, verbose=verbose, punct_labels=('p', 'punct', 'char'))
     return scorer
 
 
@@ -207,7 +212,7 @@ def write_parses(Language, dev_loc, model_dir, out_loc):
 
 
 @plac.annotations(
-    language=("The language to train", "positional", None, str, ['en','de']),
+    language=("The language to train", "positional", None, str, ['en','de', 'zh']),
     train_loc=("Location of training file or directory"),
     dev_loc=("Location of development file or directory"),
     model_dir=("Location of output model directory",),
@@ -223,7 +228,7 @@ def write_parses(Language, dev_loc, model_dir, out_loc):
 )
 def main(language, train_loc, dev_loc, model_dir, n_sents=0, n_iter=15, out_loc="", verbose=False,
          debug=False, corruption_level=0.0, gold_preproc=False, eval_only=False, pseudoprojective=False):
-    lang = {'en':English, 'de':German}.get(language)
+    lang = spacy.util.get_lang_class(language)
 
     if not eval_only:
         gold_train = list(read_json_file(train_loc))

lang_data/zh/gazetteer.json

@@ -0,0 +1,194 @@
+{
+	"Reddit": [
+		"PRODUCT",
+		{},
+		[
+			[{"lower": "reddit"}]
+		]
+	],
+	"SeptemberElevenAttacks": [
+		"EVENT",
+		{},
+		[
+			[
+				{"orth": "9/11"}
+			],
+			[
+				{"lower": "september"},
+				{"orth": "11"}
+			]
+		]
+	],
+	"Linux": [
+		"PRODUCT",
+		{},
+		[
+			[{"lower": "linux"}]
+		]
+	],
+	"Haskell": [
+		"PRODUCT",
+		{},
+		[
+			[{"lower": "haskell"}]
+		]
+	],
+	"HaskellCurry": [
+		"PERSON",
+		{},
+		[
+			[
+				{"lower": "haskell"},
+				{"lower": "curry"}
+			]
+		]
+	],
+	"Javascript": [
+		"PRODUCT",
+		{},
+		[
+			[{"lower": "javascript"}]
+		]
+	],
+	"CSS": [
+		"PRODUCT",
+		{},
+		[
+			[{"lower": "css"}],
+			[{"lower": "css3"}]
+		]
+	],
+	"displaCy": [
+		"PRODUCT",
+		{},
+		[
+			[{"lower": "displacy"}]
+		]
+	],
+	"spaCy": [
+		"PRODUCT",
+		{},
+		[
+			[{"orth": "spaCy"}]
+		]
+	],
+
+    "HTML": [
+		"PRODUCT",
+		{},
+		[
+			[{"lower": "html"}],
+			[{"lower": "html5"}]
+		]
+	],
+    "Python": [
+        "PRODUCT",
+        {},
+        [
+            [{"orth": "Python"}]
+        ]
+    ],
+    "Ruby": [
+        "PRODUCT",
+        {},
+        [
+            [{"orth": "Ruby"}]
+        ]
+    ],
+    "Digg": [
+        "PRODUCT",
+        {},
+        [
+            [{"lower": "digg"}]
+        ]
+    ],
+     "FoxNews": [
+        "ORG",
+        {},
+        [
+            [{"orth": "Fox"}],
+            [{"orth": "News"}]
+        ]
+    ],
+    "Google": [
+        "ORG",
+        {},
+        [
+            [{"lower": "google"}]
+        ]
+    ],
+    "Mac": [
+        "PRODUCT",
+        {},
+        [
+            [{"lower": "mac"}]
+        ]
+    ],
+    "Wikipedia": [
+        "PRODUCT",
+        {},
+        [
+            [{"lower": "wikipedia"}]
+        ]
+    ],
+    "Windows": [
+        "PRODUCT",
+        {},
+        [
+            [{"orth": "Windows"}]
+        ]
+    ],
+     "Dell": [
+        "ORG",
+        {},
+        [
+            [{"lower": "dell"}]
+        ]
+    ],
+    "Facebook": [
+        "ORG",
+        {},
+        [
+            [{"lower": "facebook"}]
+        ]
+    ],
+     "Blizzard": [
+        "ORG",
+        {},
+        [
+            [{"orth": "Blizzard"}]
+        ]
+    ],
+    "Ubuntu": [
+        "ORG",
+        {},
+        [
+            [{"orth": "Ubuntu"}]
+        ]
+    ],
+    "Youtube": [
+        "PRODUCT",
+        {},
+        [
+            [{"lower": "youtube"}]
+        ]
+    ],
+    "false_positives": [
+        null,
+        {},
+        [
+            [{"orth": "Shit"}],
+            [{"orth": "Weed"}],
+            [{"orth": "Cool"}],
+            [{"orth": "Btw"}],
+            [{"orth": "Bah"}],
+            [{"orth": "Bullshit"}],
+            [{"orth": "Lol"}],
+            [{"orth": "Yo"}, {"lower": "dawg"}],
+            [{"orth": "Yay"}],
+            [{"orth": "Ahh"}],
+            [{"orth": "Yea"}],
+            [{"orth": "Bah"}]
+        ]
+    ]
+}

lang_data/zh/infix.txt

@@ -0,0 +1,6 @@
+\.\.\.
+(?<=[a-z])\.(?=[A-Z])
+(?<=[a-zA-Z])-(?=[a-zA-z])
+(?<=[a-zA-Z])--(?=[a-zA-z])
+(?<=[0-9])-(?=[0-9])
+(?<=[A-Za-z]),(?=[A-Za-z])

lang_data/zh/morphs.json

@@ -0,0 +1 @@
+{}

lang_data/zh/prefix.txt

@@ -0,0 +1,21 @@
+,
+"
+(
+[
+{
+*
+<
+$
+£
+“
+'
+``
+`
+#
+US$
+C$
+A$
+a-
+‘
+....
+...

lang_data/zh/specials.json

@@ -0,0 +1 @@
+{}

lang_data/zh/suffix.txt

@@ -0,0 +1,26 @@
+,
+\"
+\)
+\]
+\}
+\*
+\!
+\?
+%
+\$
+>
+:
+;
+'
+”
+''
+'s
+'S
+’s
+’S
+’
+\.\.
+\.\.\.
+\.\.\.\.
+(?<=[a-z0-9)\]"'%\)])\.
+(?<=[0-9])km

lang_data/zh/tag_map.json

@@ -0,0 +1,43 @@
+{
+    "NR":   {"pos": "PROPN"},
+    "AD":   {"pos": "ADV"},
+    "NN":   {"pos": "NOUN"},
+    "CD":   {"pos": "NUM"},
+    "DEG":  {"pos": "PART"},
+    "PN":   {"pos": "PRON"},
+    "M":    {"pos": "PART"},
+    "JJ":   {"pos": "ADJ"},
+    "DEC":  {"pos": "PART"},
+    "NT":   {"pos": "NOUN"},
+    "DT":   {"pos": "DET"},
+    "LC":   {"pos": "PART"},
+    "CC":   {"pos": "CONJ"},
+    "AS":   {"pos": "PART"},
+    "SP":   {"pos": "PART"},
+    "IJ":   {"pos": "INTJ"},
+    "OD":   {"pos": "NUM"},
+    "MSP":  {"pos": "PART"},
+    "CS":   {"pos": "SCONJ"},
+    "ETC":  {"pos": "PART"},
+    "DEV":  {"pos": "PART"},
+    "BA":   {"pos": "AUX"},
+    "SB":   {"pos": "AUX"},
+    "DER":  {"pos": "PART"},
+    "LB":   {"pos": "AUX"},
+    "P":    {"pos": "ADP"},
+    "URL":  {"pos": "SYM"},
+    "FRAG": {"pos": "X"},
+    "X":    {"pos": "X"},
+    "ON":   {"pos": "X"},
+    "FW":   {"pos": "X"},
+    "VC":   {"pos": "VERB"},
+    "VV":   {"pos": "VERB"},
+    "VA":   {"pos": "VERB"},
+    "VE":   {"pos": "VERB"},
+    "PU":   {"pos": "PUNCT"},
+    "SP":   {"pos": "SPACE"},
+    "NP":   {"pos": "X"},
+    "_":    {"pos": "X"},
+    "VP":   {"pos": "X"},
+    "CHAR": {"pos": "X"}
+}

requirements.txt

@@ -10,3 +10,4 @@ six
 ujson
 cloudpickle
 sputnik>=0.9.2,<0.10.0
+jieba

setup.py

@@ -193,7 +193,8 @@ def setup_package():
                 'plac',
                 'six',
                 'cloudpickle',
-                'sputnik>=0.9.2,<0.10.0'],
+                'sputnik>=0.9.2,<0.10.0',
+                'jieba'],
             classifiers=[
                 'Development Status :: 5 - Production/Stable',
                 'Environment :: Console',

spacy/__init__.py

@@ -2,14 +2,26 @@ from .util import set_lang_class, get_lang_class, get_package, get_package_by_na
 
 from . import en
 from . import de
+from . import zh
 
 
 set_lang_class(en.English.lang, en.English)
 set_lang_class(de.German.lang, de.German)
+set_lang_class(zh.Chinese.lang, zh.Chinese)
 
 
-def load(name, vectors=None, via=None):
+def load(name, vocab=None, tokenizer=None, parser=None, tagger=None, entity=None,
+         matcher=None, serializer=None, vectors=None, via=None):
     package = get_package_by_name(name, via=via)
     vectors_package = get_package_by_name(vectors, via=via)
     cls = get_lang_class(name)
-    return cls(package=package, vectors_package=vectors_package)
+    return cls(
+        package=package,
+        vectors_package=vectors_package,
+        vocab=vocab,
+        tokenizer=tokenizer,
+        tagger=tagger,
+        parser=parser,
+        entity=entity,
+        matcher=matcher,
+        serializer=serializer)

spacy/gold.pyx

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 import numpy
 import io
 import json
@@ -42,7 +43,6 @@ def tags_to_entities(tags):
     return entities
 
 
-
 def align(cand_words, gold_words):
     cost, edit_path = _min_edit_path(cand_words, gold_words)
     alignment = []
@@ -63,7 +63,7 @@ def align(cand_words, gold_words):
     return alignment
 
 
-punct_re = re.compile(r'\W')
+punct_re = re.compile(r'\W', re.UNICODE)
 def _min_edit_path(cand_words, gold_words):
     cdef:
         Pool mem
@@ -264,13 +264,3 @@ cdef class GoldParse:
 
 def is_punct_label(label):
     return label == 'P' or label.lower() == 'punct'
-
-
-
-
-
-
-
-
-
-

spacy/language.py

@@ -135,6 +135,10 @@ class Language(object):
         else:
             return None
 
+    @classmethod
+    def default_tokenizer(cls, package, vocab):
+        return Tokenizer.from_package(package, vocab)
+
     def __init__(self,
         data_dir=None,
         vocab=None,
@@ -178,7 +182,7 @@ class Language(object):
             vocab = self.default_vocab(package, vectors_package=vectors_package)
         self.vocab = vocab
         if tokenizer in (None, True):
-            tokenizer = Tokenizer.from_package(package, self.vocab)
+            tokenizer = self.default_tokenizer(package, self.vocab)
         self.tokenizer = tokenizer
         if tagger in (None, True):
             tagger = Tagger.from_package(package, self.vocab)

spacy/matcher.pyx

@@ -163,12 +163,12 @@ cdef class Matcher:
         patterns = package.load_json(('vocab', 'gazetteer.json'))
         return cls(vocab, patterns)
 
-    def __init__(self, vocab, patterns):
+    def __init__(self, vocab, patterns={}):
+        self._patterns = dict(patterns) # Make sure we own the object
         self.vocab = vocab
         self.mem = Pool()
         self.vocab = vocab
-        self._patterns = dict(patterns)
-        for entity_key, (etype, attrs, specs) in sorted(patterns.items()):
+        for entity_key, (etype, attrs, specs) in sorted(self._patterns.items()):
             self.add(entity_key, etype, attrs, specs)
 
     def __reduce__(self):

spacy/tests/gold/test_alignment.py

@@ -0,0 +1,15 @@
+from spacy.gold import _min_edit_path
+
+
+def test_min_edit_path():
+    '''Test problem that arose from Chinese parsing, where alignment didn't match
+    at the start, depending on which direction followed. The solution was that
+    a regular expression did not have re.UNICODE flag, causing it to over match.
+    '''
+
+    cand_words = [u'\u53cc\u65b9', u'D', u'-', u'RAM']
+    gold_words = [u'\u53cc\u65b9', u'D-RAM']
+    cost, alignment = _min_edit_path(cand_words, gold_words)
+    assert alignment[0] == 'M'
+    cost, alignment = _min_edit_path(gold_words, cand_words)
+    assert alignment[0] == 'M'

spacy/tokens/doc.pyx

@@ -82,6 +82,13 @@ cdef class Doc:
         self._py_tokens = []
         self._vector = None
         self.noun_chunks_iterator = DocIterator(self)
+        cdef unicode orth
+        cdef bint has_space
+        cdef const LexemeC* lex
+        if orths_and_spaces:
+            for orth, has_space in orths_and_spaces:
+                lex = self.vocab.get(self.mem, orth)
+                self.push_back(lex, has_space)
 
     def __getitem__(self, object i):
         """Get a Token or a Span from the Doc.

spacy/zh/__init__.py

@@ -0,0 +1,33 @@
+import jieba
+from ..language import Language
+
+from ..tokenizer import Tokenizer
+from ..tokens.doc import Doc
+
+
+class JiebaTokenizer(Tokenizer):
+    def __call__(self, text):
+        orths = []
+        spaces = []
+        for orth, start, end in jieba.tokenize(text):
+            # TODO: This is wrong if multiple spaces in a row.
+            if orth == u' ':
+                spaces[-1] = True
+            else:
+                orths.append(orth)
+                spaces.append(False)
+        return Doc(self.vocab, orths_and_spaces=zip(orths, spaces))
+
+
+class CharacterTokenizer(Tokenizer):
+    def __call__(self, text):
+        return self.tokens_from_list(list(text))
+
+
+class Chinese(Language):
+    lang = u'zh'
+
+    @classmethod
+    def default_tokenizer(cls, package, vocab):
+        '''Return Jieba-wrapper tokenizer.'''
+        return JiebaTokenizer.from_package(package, vocab)