Update PyTorch section

examples/training/train_pytorch_textcat.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+# coding: utf8
+"""Define a text classification model using PyTorch, and wrap it with Thinc's
+PytorchWrapper class, so it can be used in spaCy's TextCategorizer component.
+
+The model is added to spacy.pipeline, and predictions are available via
+`doc.cats`. For more details, see the documentation:
+
+* Deep learning: https://alpha.spacy.io/usage/deep-learning
+* Text classification: https://alpha.spacy.io/usage/text-classification
+
+Developed for: spaCy 2.0.0a19
+Last updated for: spaCy 2.0.0a19
+"""
+from __future__ import unicode_literals, print_function
+import plac
+import random
+from pathlib import Path
+import thinc.extra.datasets
+import thinc.extra.wrappers
+
+import spacy
+from spacy.gold import GoldParse, minibatch
+from spacy.util import compounding
+
+
+@plac.annotations(
+    model=("Model name. Defaults to blank 'en' model.", "option", "m", str),
+    output_dir=("Optional output directory", "option", "o", Path),
+    n_texts=("Number of texts to train from", "option", "t", int),
+    n_iter=("Number of training iterations", "option", "n", int))
+def main(model=None, output_dir=None, n_iter=20, n_texts=2000):
+    if model is not None:
+        nlp = spacy.load(model)  # load existing spaCy model
+        print("Loaded model '%s'" % model)
+    else:
+        nlp = spacy.blank('en')  # create blank Language class
+        print("Created blank 'en' model")
+
+    # Create the PyTorch neural network model, and wrap it with Thinc. This
+    # gives it the API that spaCy expects.
+    pt_model = create_model()
+    textcat = thinc.extra.wrappers.PyTorchWrapper(pt_model)
+    nlp.add_pipe(textcat, last=True)
+
+    # add label to text classifier
+    textcat.add_label('POSITIVE')
+
+    # load the IMBD dataset
+    print("Loading IMDB data...")
+    (train_texts, train_cats), (dev_texts, dev_cats) = load_data(limit=n_texts)
+    print("Using %d training examples" % n_texts)
+    train_docs = [nlp.tokenizer(text) for text in train_texts]
+    train_gold = [GoldParse(doc, cats=cats) for doc, cats in
+                  zip(train_docs, train_cats)]
+    train_data = list(zip(train_docs, train_gold))
+
+    # get names of other pipes to disable them during training
+    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'textcat']
+    with nlp.disable_pipes(*other_pipes):  # only train textcat
+        optimizer = nlp.begin_training()
+        print("Training the model...")
+        print('{:^5}\t{:^5}\t{:^5}\t{:^5}'.format('LOSS', 'P', 'R', 'F'))
+        for i in range(n_iter):
+            losses = {}
+            # batch up the examples using spaCy's minibatch
+            batches = minibatch(train_data, size=compounding(4., 32., 1.001))
+            for batch in batches:
+                docs, golds = zip(*batch)
+                nlp.update(docs, golds, sgd=optimizer, drop=0.2, losses=losses)
+            with textcat.model.use_params(optimizer.averages):
+                # evaluate on the dev data split off in load_data()
+                scores = evaluate(nlp.tokenizer, textcat, dev_texts, dev_cats)
+            print('{0:.3f}\t{1:.3f}\t{2:.3f}\t{3:.3f}'  # print a simple table
+                  .format(losses['textcat'], scores['textcat_p'],
+                          scores['textcat_r'], scores['textcat_f']))
+
+    # test the trained model
+    test_text = "This movie sucked"
+    doc = nlp(test_text)
+    print(test_text, doc.cats)
+
+    if output_dir is not None:
+        output_dir = Path(output_dir)
+        if not output_dir.exists():
+            output_dir.mkdir()
+        nlp.to_disk(output_dir)
+        print("Saved model to", output_dir)
+
+        # test the saved model
+        print("Loading from", output_dir)
+        nlp2 = spacy.load(output_dir)
+        doc2 = nlp2(test_text)
+        print(test_text, doc2.cats)
+
+
+def load_data(limit=0, split=0.8):
+    """Load data from the IMDB dataset."""
+    # Partition off part of the train data for evaluation
+    train_data, _ = thinc.extra.datasets.imdb()
+    random.shuffle(train_data)
+    train_data = train_data[-limit:]
+    texts, labels = zip(*train_data)
+    cats = [{'POSITIVE': bool(y)} for y in labels]
+    split = int(len(train_data) * split)
+    return (texts[:split], cats[:split]), (texts[split:], cats[split:])
+
+
+def evaluate(tokenizer, textcat, texts, cats):
+    docs = (tokenizer(text) for text in texts)
+    tp = 1e-8  # True positives
+    fp = 1e-8  # False positives
+    fn = 1e-8  # False negatives
+    tn = 1e-8  # True negatives
+    for i, doc in enumerate(textcat.pipe(docs)):
+        gold = cats[i]
+        for label, score in doc.cats.items():
+            if label not in gold:
+                continue
+            if score >= 0.5 and gold[label] >= 0.5:
+                tp += 1.
+            elif score >= 0.5 and gold[label] < 0.5:
+                fp += 1.
+            elif score < 0.5 and gold[label] < 0.5:
+                tn += 1
+            elif score < 0.5 and gold[label] >= 0.5:
+                fn += 1
+    precision = tp / (tp + fp)
+    recall = tp / (tp + fn)
+    f_score = 2 * (precision * recall) / (precision + recall)
+    return {'textcat_p': precision, 'textcat_r': recall, 'textcat_f': f_score}
+
+
+if __name__ == '__main__':
+    plac.call(main)

website/assets/js/vendor/prism.min.js

@@ -16,7 +16,7 @@ Prism.languages.json={property:/".*?"(?=\s*:)/gi,string:/"(?!:)(\\?[^"])*?"(?!:)
 !function(a){var e=/\\([^a-z()[\]]|[a-z\*]+)/i,n={"equation-command":{pattern:e,alias:"regex"}};a.languages.latex={comment:/%.*/m,cdata:{pattern:/(\\begin\{((?:verbatim|lstlisting)\*?)\})([\w\W]*?)(?=\\end\{\2\})/,lookbehind:!0},equation:[{pattern:/\$(?:\\?[\w\W])*?\$|\\\((?:\\?[\w\W])*?\\\)|\\\[(?:\\?[\w\W])*?\\\]/,inside:n,alias:"string"},{pattern:/(\\begin\{((?:equation|math|eqnarray|align|multline|gather)\*?)\})([\w\W]*?)(?=\\end\{\2\})/,lookbehind:!0,inside:n,alias:"string"}],keyword:{pattern:/(\\(?:begin|end|ref|cite|label|usepackage|documentclass)(?:\[[^\]]+\])?\{)[^}]+(?=\})/,lookbehind:!0},url:{pattern:/(\\url\{)[^}]+(?=\})/,lookbehind:!0},headline:{pattern:/(\\(?:part|chapter|section|subsection|frametitle|subsubsection|paragraph|subparagraph|subsubparagraph|subsubsubparagraph)\*?(?:\[[^\]]+\])?\{)[^}]+(?=\}(?:\[[^\]]+\])?)/,lookbehind:!0,alias:"class-name"},"function":{pattern:e,alias:"selector"},punctuation:/[[\]{}&]/}}(Prism);
 Prism.languages.makefile={comment:{pattern:/(^|[^\\])#(?:\\(?:\r\n|[\s\S])|.)*/,lookbehind:!0},string:/(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,builtin:/\.[A-Z][^:#=\s]+(?=\s*:(?!=))/,symbol:{pattern:/^[^:=\r\n]+(?=\s*:(?!=))/m,inside:{variable:/\$+(?:[^(){}:#=\s]+|(?=[({]))/}},variable:/\$+(?:[^(){}:#=\s]+|\([@*%<^+?][DF]\)|(?=[({]))/,keyword:[/-include\b|\b(?:define|else|endef|endif|export|ifn?def|ifn?eq|include|override|private|sinclude|undefine|unexport|vpath)\b/,{pattern:/(\()(?:addsuffix|abspath|and|basename|call|dir|error|eval|file|filter(?:-out)?|findstring|firstword|flavor|foreach|guile|if|info|join|lastword|load|notdir|or|origin|patsubst|realpath|shell|sort|strip|subst|suffix|value|warning|wildcard|word(?:s|list)?)(?=[ \t])/,lookbehind:!0}],operator:/(?:::|[?:+!])?=|[|@]/,punctuation:/[:;(){}]/};
 Prism.languages.markdown=Prism.languages.extend("markup",{}),Prism.languages.insertBefore("markdown","prolog",{blockquote:{pattern:/^>(?:[\t ]*>)*/m,alias:"punctuation"},code:[{pattern:/^(?: {4}|\t).+/m,alias:"keyword"},{pattern:/``.+?``|`[^`\n]+`/,alias:"keyword"}],title:[{pattern:/\w+.*(?:\r?\n|\r)(?:==+|--+)/,alias:"important",inside:{punctuation:/==+$|--+$/}},{pattern:/(^\s*)#+.+/m,lookbehind:!0,alias:"important",inside:{punctuation:/^#+|#+$/}}],hr:{pattern:/(^\s*)([*-])([\t ]*\2){2,}(?=\s*$)/m,lookbehind:!0,alias:"punctuation"},list:{pattern:/(^\s*)(?:[*+-]|\d+\.)(?=[\t ].)/m,lookbehind:!0,alias:"punctuation"},"url-reference":{pattern:/!?\[[^\]]+\]:[\t ]+(?:\S+|<(?:\\.|[^>\\])+>)(?:[\t ]+(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\)))?/,inside:{variable:{pattern:/^(!?\[)[^\]]+/,lookbehind:!0},string:/(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\))$/,punctuation:/^[\[\]!:]|[<>]/},alias:"url"},bold:{pattern:/(^|[^\\])(\*\*|__)(?:(?:\r?\n|\r)(?!\r?\n|\r)|.)+?\2/,lookbehind:!0,inside:{punctuation:/^\*\*|^__|\*\*$|__$/}},italic:{pattern:/(^|[^\\])([*_])(?:(?:\r?\n|\r)(?!\r?\n|\r)|.)+?\2/,lookbehind:!0,inside:{punctuation:/^[*_]|[*_]$/}},url:{pattern:/!?\[[^\]]+\](?:\([^\s)]+(?:[\t ]+"(?:\\.|[^"\\])*")?\)| ?\[[^\]\n]*\])/,inside:{variable:{pattern:/(!?\[)[^\]]+(?=\]$)/,lookbehind:!0},string:{pattern:/"(?:\\.|[^"\\])*"(?=\)$)/}}}}),Prism.languages.markdown.bold.inside.url=Prism.util.clone(Prism.languages.markdown.url),Prism.languages.markdown.italic.inside.url=Prism.util.clone(Prism.languages.markdown.url),Prism.languages.markdown.bold.inside.italic=Prism.util.clone(Prism.languages.markdown.italic),Prism.languages.markdown.italic.inside.bold=Prism.util.clone(Prism.languages.markdown.bold);
-Prism.languages.python={"triple-quoted-string":{pattern:/"""[\s\S]+?"""|'''[\s\S]+?'''/,alias:"string"},comment:{pattern:/(^|[^\\])#.*/,lookbehind:!0},string:/("|')(?:\\?.)*?\1/,"function":{pattern:/((?:^|\s)def[ \t]+)[a-zA-Z_][a-zA-Z0-9_]*(?=\()/g,lookbehind:!0},"class-name":{pattern:/(\bclass\s+)[a-z0-9_]+/i,lookbehind:!0},keyword:/\b(?:as|assert|async|await|break|class|continue|def|del|elif|else|except|exec|finally|for|from|global|if|import|in|is|lambda|pass|print|raise|return|try|while|with|yield)\b/,"boolean":/\b(?:True|False)\b/,number:/\b-?(?:0[bo])?(?:(?:\d|0x[\da-f])[\da-f]*\.?\d*|\.\d+)(?:e[+-]?\d+)?j?\b/i,operator:/[-+%=]=?|!=|\*\*?=?|\/\/?=?|<[<=>]?|>[=>]?|[&|^~]|\b(?:or|and|not)\b/,punctuation:/[{}[\];(),.:]/};
+Prism.languages.python={"triple-quoted-string":{pattern:/"""[\s\S]+?"""|'''[\s\S]+?'''/,alias:"string"},comment:{pattern:/(^|[^\\])#.*/,lookbehind:!0},string:/("|')(?:\\?.)*?\1/,"function":{pattern:/((?:^|\s)def[ \t]+)[a-zA-Z_][a-zA-Z0-9_]*(?=\()/g,lookbehind:!0},"class-name":{pattern:/(\bclass\s+)[a-z0-9_]+/i,lookbehind:!0},keyword:/\b(?:as|assert|async|await|break|class|continue|def|del|elif|else|except|exec|finally|for|from|global|if|import|in|is|lambda|pass|print|raise|return|try|while|with|yield)\b/,"boolean":/\b(?:True|False)\b/,number:/\b-?(?:0[bo])?(?:(?:\d|0x[\da-f])[\da-f]*\.?\d*|\.\d+)(?:e[+-]?\d+)?j?\b/i,operator:/[-+%=]=?|!=|\*\*?=?|\/\/?=?|<[<=>]?|>[=>]?|[&|^~]|\b(?:or|and|not)\b/,punctuation:/[{}[\];(),.:]/,"constant":/\b[A-Z_]{2,}\b/};
 Prism.languages.rest={table:[{pattern:/(\s*)(?:\+[=-]+)+\+(?:\r?\n|\r)(?:\1(?:[+|].+)+[+|](?:\r?\n|\r))+\1(?:\+[=-]+)+\+/,lookbehind:!0,inside:{punctuation:/\||(?:\+[=-]+)+\+/}},{pattern:/(\s*)(?:=+ +)+=+((?:\r?\n|\r)\1.+)+(?:\r?\n|\r)\1(?:=+ +)+=+(?=(?:\r?\n|\r){2}|\s*$)/,lookbehind:!0,inside:{punctuation:/[=-]+/}}],"substitution-def":{pattern:/(^\s*\.\. )\|(?:[^|\s](?:[^|]*[^|\s])?)\| [^:]+::/m,lookbehind:!0,inside:{substitution:{pattern:/^\|(?:[^|\s]|[^|\s][^|]*[^|\s])\|/,alias:"attr-value",inside:{punctuation:/^\||\|$/}},directive:{pattern:/( +)[^:]+::/,lookbehind:!0,alias:"function",inside:{punctuation:/::$/}}}},"link-target":[{pattern:/(^\s*\.\. )\[[^\]]+\]/m,lookbehind:!0,alias:"string",inside:{punctuation:/^\[|\]$/}},{pattern:/(^\s*\.\. )_(?:`[^`]+`|(?:[^:\\]|\\.)+):/m,lookbehind:!0,alias:"string",inside:{punctuation:/^_|:$/}}],directive:{pattern:/(^\s*\.\. )[^:]+::/m,lookbehind:!0,alias:"function",inside:{punctuation:/::$/}},comment:{pattern:/(^\s*\.\.)(?:(?: .+)?(?:(?:\r?\n|\r).+)+| .+)(?=(?:\r?\n|\r){2}|$)/m,lookbehind:!0},title:[{pattern:/^(([!"#$%&'()*+,\-.\/:;<=>?@\[\\\]^_`{|}~])\2+)(?:\r?\n|\r).+(?:\r?\n|\r)\1$/m,inside:{punctuation:/^[!"#$%&'()*+,\-.\/:;<=>?@\[\\\]^_`{|}~]+|[!"#$%&'()*+,\-.\/:;<=>?@\[\\\]^_`{|}~]+$/,important:/.+/}},{pattern:/(^|(?:\r?\n|\r){2}).+(?:\r?\n|\r)([!"#$%&'()*+,\-.\/:;<=>?@\[\\\]^_`{|}~])\2+(?=\r?\n|\r|$)/,lookbehind:!0,inside:{punctuation:/[!"#$%&'()*+,\-.\/:;<=>?@\[\\\]^_`{|}~]+$/,important:/.+/}}],hr:{pattern:/((?:\r?\n|\r){2})([!"#$%&'()*+,\-.\/:;<=>?@\[\\\]^_`{|}~])\2{3,}(?=(?:\r?\n|\r){2})/,lookbehind:!0,alias:"punctuation"},field:{pattern:/(^\s*):[^:\r\n]+:(?= )/m,lookbehind:!0,alias:"attr-name"},"command-line-option":{pattern:/(^\s*)(?:[+-][a-z\d]|(?:\-\-|\/)[a-z\d-]+)(?:[ =](?:[a-z][a-z\d_-]*|<[^<>]+>))?(?:, (?:[+-][a-z\d]|(?:\-\-|\/)[a-z\d-]+)(?:[ =](?:[a-z][a-z\d_-]*|<[^<>]+>))?)*(?=(?:\r?\n|\r)? {2,}\S)/im,lookbehind:!0,alias:"symbol"},"literal-block":{pattern:/::(?:\r?\n|\r){2}([ \t]+).+(?:(?:\r?\n|\r)\1.+)*/,inside:{"literal-block-punctuation":{pattern:/^::/,alias:"punctuation"}}},"quoted-literal-block":{pattern:/::(?:\r?\n|\r){2}([!"#$%&'()*+,\-.\/:;<=>?@\[\\\]^_`{|}~]).*(?:(?:\r?\n|\r)\1.*)*/,inside:{"literal-block-punctuation":{pattern:/^(?:::|([!"#$%&'()*+,\-.\/:;<=>?@\[\\\]^_`{|}~])\1*)/m,alias:"punctuation"}}},"list-bullet":{pattern:/(^\s*)(?:[*+\-•‣⁃]|\(?(?:\d+|[a-z]|[ivxdclm]+)\)|(?:\d+|[a-z]|[ivxdclm]+)\.)(?= )/im,lookbehind:!0,alias:"punctuation"},"doctest-block":{pattern:/(^\s*)>>> .+(?:(?:\r?\n|\r).+)*/m,lookbehind:!0,inside:{punctuation:/^>>>/}},inline:[{pattern:/(^|[\s\-:\/'"<(\[{])(?::[^:]+:`.*?`|`.*?`:[^:]+:|(\*\*?|``?|\|)(?!\s).*?[^\s]\2(?=[\s\-.,:;!?\\\/'")\]}]|$))/m,lookbehind:!0,inside:{bold:{pattern:/(^\*\*).+(?=\*\*$)/,lookbehind:!0},italic:{pattern:/(^\*).+(?=\*$)/,lookbehind:!0},"inline-literal":{pattern:/(^``).+(?=``$)/,lookbehind:!0,alias:"symbol"},role:{pattern:/^:[^:]+:|:[^:]+:$/,alias:"function",inside:{punctuation:/^:|:$/}},"interpreted-text":{pattern:/(^`).+(?=`$)/,lookbehind:!0,alias:"attr-value"},substitution:{pattern:/(^\|).+(?=\|$)/,lookbehind:!0,alias:"attr-value"},punctuation:/\*\*?|``?|\|/}}],link:[{pattern:/\[[^\]]+\]_(?=[\s\-.,:;!?\\\/'")\]}]|$)/,alias:"string",inside:{punctuation:/^\[|\]_$/}},{pattern:/(?:\b[a-z\d](?:[_.:+]?[a-z\d]+)*_?_|`[^`]+`_?_|_`[^`]+`)(?=[\s\-.,:;!?\\\/'")\]}]|$)/i,alias:"string",inside:{punctuation:/^_?`|`$|`?_?_$/}}],punctuation:{pattern:/(^\s*)(?:\|(?= |$)|(?:---?|—|\.\.|__)(?= )|\.\.$)/m,lookbehind:!0}};
 !function(e){e.languages.sass=e.languages.extend("css",{comment:{pattern:/^([ \t]*)\/[\/*].*(?:(?:\r?\n|\r)\1[ \t]+.+)*/m,lookbehind:!0}}),e.languages.insertBefore("sass","atrule",{"atrule-line":{pattern:/^(?:[ \t]*)[@+=].+/m,inside:{atrule:/(?:@[\w-]+|[+=])/m}}}),delete e.languages.sass.atrule;var a=/((\$[-_\w]+)|(#\{\$[-_\w]+\}))/i,t=[/[+*\/%]|[=!]=|<=?|>=?|\b(?:and|or|not)\b/,{pattern:/(\s+)-(?=\s)/,lookbehind:!0}];e.languages.insertBefore("sass","property",{"variable-line":{pattern:/^[ \t]*\$.+/m,inside:{punctuation:/:/,variable:a,operator:t}},"property-line":{pattern:/^[ \t]*(?:[^:\s]+ *:.*|:[^:\s]+.*)/m,inside:{property:[/[^:\s]+(?=\s*:)/,{pattern:/(:)[^:\s]+/,lookbehind:!0}],punctuation:/:/,variable:a,operator:t,important:e.languages.sass.important}}}),delete e.languages.sass.property,delete e.languages.sass.important,delete e.languages.sass.selector,e.languages.insertBefore("sass","punctuation",{selector:{pattern:/([ \t]*)\S(?:,?[^,\r\n]+)*(?:,(?:\r?\n|\r)\1[ \t]+\S(?:,?[^,\r\n]+)*)*/,lookbehind:!0}})}(Prism);
 Prism.languages.scss=Prism.languages.extend("css",{comment:{pattern:/(^|[^\\])(?:\/\*[\w\W]*?\*\/|\/\/.*)/,lookbehind:!0},atrule:{pattern:/@[\w-]+(?:\([^()]+\)|[^(])*?(?=\s+[{;])/,inside:{rule:/@[\w-]+/}},url:/(?:[-a-z]+-)*url(?=\()/i,selector:{pattern:/(?=\S)[^@;\{\}\(\)]?([^@;\{\}\(\)]|&|#\{\$[-_\w]+\})+(?=\s*\{(\}|\s|[^\}]+(:|\{)[^\}]+))/m,inside:{placeholder:/%[-_\w]+/}}}),Prism.languages.insertBefore("scss","atrule",{keyword:[/@(?:if|else(?: if)?|for|each|while|import|extend|debug|warn|mixin|include|function|return|content)/i,{pattern:/( +)(?:from|through)(?= )/,lookbehind:!0}]}),Prism.languages.insertBefore("scss","property",{variable:/\$[-_\w]+|#\{\$[-_\w]+\}/}),Prism.languages.insertBefore("scss","function",{placeholder:{pattern:/%[-_\w]+/,alias:"selector"},statement:/\B!(?:default|optional)\b/i,"boolean":/\b(?:true|false)\b/,"null":/\bnull\b/,operator:{pattern:/(\s)(?:[-+*\/%]|[=!]=|<=?|>=?|and|or|not)(?=\s)/,lookbehind:!0}}),Prism.languages.scss.atrule.inside.rest=Prism.util.clone(Prism.languages.scss);

website/usage/_data.json

@@ -128,7 +128,7 @@
         "next": "training",
         "menu": {
             "Pre-processing Text": "pre-processing",
-            "spaCy and Thinc": "thinc",
+            "Wrapping Models": "wrapping",
             "TensorFlow / Keras": "tensorflow-keras",
             "scikit-learn": "scikit-learn",
             "PyTorch": "pytorch",

website/usage/_deep-learning/_pytorch.jade

@@ -9,83 +9,8 @@
     |  to create spaCy pipeline components, to add annotations to the
     |  #[code Doc] object.
 
-+under-construction
-
-p
-    |  Here's how a #[code begin_update] function that wraps an arbitrary
-    |  PyTorch model would look:
-
 +code.
-    class PytorchWrapper(thinc.neural.Model):
-        def __init__(self, pytorch_model):
-            self.pytorch_model = pytorch_model
+    from thinc.extra.wrappers import PyTorchWrapper
+    model = PyTorchWrapper(YOUR_PYTORCH_MODEL)
 
-        def begin_update(self, x_data, drop=0.):
-            x_var = Variable(x_data)
-            # Make prediction
-            y_var = pytorch_model.forward(x_var)
-            def backward(dy_data, sgd=None):
-                dy_var = Variable(dy_data)
-                dx_var = torch.autograd.backward(x_var, dy_var)
-                return dx_var
-            return y_var.data, backward
-
-p
-    |  PyTorch requires data to be wrapped in a container, #[code Variable],
-    |  that tracks the operations performed on the data. This "tape" of
-    |  operations is then used by #[code torch.autograd.backward] to compute the
-    |  gradient with respect to the input. For example, the following code
-    |  constructs a PyTorch Linear layer that takes a vector of shape
-    |  #[code (length, 2)], multiples it by a #[code (2, 2)] matrix of weights,
-    |  adds a #[code (2,)] bias, and returns the resulting #[code (length, 2)]
-    |  vector:
-
-+code("PyTorch Linear").
-    from torch import autograd
-    from torch import nn
-    import torch
-    import numpy
-
-    pt_model = nn.Linear(2, 2)
-    length = 5
-
-    input_data = numpy.ones((5, 2), dtype='f')
-    input_var = autograd.Variable(torch.Tensor(input_data))
-
-    output_var = pt_model(input_var)
-    output_data = output_var.data.numpy()
-
-p
-    |  Given target values we would like the output data to approximate, we can
-    |  then "learn" values of the parameters within #[code pt_model], to give us
-    |  output that's closer to our target. As a trivial example, let's make the
-    |  linear layer compute the negative inverse of the input:
-
-+code.
-    def get_target(input_data):
-        return -(1 / input_data)
-
-p
-    |  To update the PyTorch model, we create an optimizer and give it
-    |  references to the model's parameters. We'll then randomly generate input
-    |  data and get the target result we'd like the function to produce. We then
-    |  compute the #[strong gradient of the error] between the current output
-    |  and the target. Using the most popular definition of "error", this is
-    |  simply the average difference:
-
-+code.
-    from torch import optim
-
-    optimizer = optim.SGD(pt_model.parameters(), lr = 0.01)
-    for i in range(10):
-        input_data = numpy.random.uniform(-1., 1., (length, 2))
-        target = -(1 / input_data)
-
-        output_var = pt_model(autograd.Variable(torch.Tensor(input_data)))
-        output_data = output_var.data.numpy()
-
-        d_output_data = (output_data - target) / length
-        d_output_var = autograd.Variable(torch.Tensor(d_output_data))
-
-        d_input_var = torch.autograg.backward(output_var, d_output_var)
-        optimizer.step()
++github("spacy", "examples/training/train_pytorch_textcat.py")

website/usage/_deep-learning/_thinc.jade

@@ -1,66 +0,0 @@
-//- 💫 DOCS > USAGE > DEEP LEARNING > THINC
-
-p
-    |  #[+a(gh("thinc")) Thinc] is the machine learning library powering spaCy.
-    |  It's a practical toolkit for implementing models that follow the
-    |  #[+a("https://explosion.ai/blog/deep-learning-formula-nlp", true) "Embed, encode, attend, predict"]
-    |  architecture. It's designed to be easy to install, efficient for CPU
-    |  usage and optimised for NLP and deep learning with text – in particular,
-    |  hierarchically structured input and variable-length sequences.
-
-p
-    |  spaCy's built-in pipeline components can all be powered by any object
-    |  that follows Thinc's #[code Model] API. If a wrapper is not yet available
-    |  for the library you're using, you should create a
-    |  #[code thinc.neural.Model] subclass that implements a #[code begin_update]
-    |  method. You'll also want to implement #[code to_bytes], #[code from_bytes],
-    |  #[code to_disk] and #[code from_disk] methods, to save and load your
-    |  model. Here's the tempate you'll need to fill in:
-
-    +code("Thinc Model API").
-        class ThincModel(thinc.neural.Model):
-            def __init__(self, *args, **kwargs):
-                pass
-
-            def begin_update(self, X, drop=0.):
-                def backprop(dY, sgd=None):
-                    return dX
-                return Y, backprop
-
-            def to_disk(self, path, **exclude):
-                return None
-
-            def from_disk(self, path, **exclude):
-                return self
-
-            def to_bytes(self, **exclude):
-                return bytes
-
-            def from_bytes(self, msgpacked_bytes, **exclude):
-                return self
-
-p
-    |  The #[code begin_update] method should return a callback, that takes the
-    |  gradient with respect to the output, and returns the gradient with
-    |  respect to the input.  It's usually convenient to implement the callback
-    |  as a nested function, so you can refer to any intermediate variables from
-    |  the forward computation in the enclosing scope.
-
-+h(3, "how-thinc-works") How Thinc works
-
-p
-    |  Neural networks are all about composing small functions that we know how
-    |  to differentiate into larger functions that we know how to differentiate.
-    |  To differentiate a function efficiently, you usually need to store
-    |  intermediate results, computed during the "forward pass", to reuse them
-    |  during the backward pass. Most libraries require the data passed through
-    |  the network to accumulate these intermediate result. This is the "tape"
-    |  in tape-based differentiation.
-
-p
-    |  In Thinc, a model that computes #[code y = f(x)] is required to also
-    |  return a callback that computes #[code dx = f'(dy)]. The same
-    |  intermediate state needs to be tracked, but this becomes an
-    |  implementation detail for the model to take care of – usually, the
-    |  callback is implemented as a closure, so the intermediate results can be
-    |  read from the enclosing scope.

website/usage/_deep-learning/_wrapping.jade

@@ -0,0 +1,127 @@
+//- 💫 DOCS > USAGE > DEEP LEARNING > WRAPPING MODELS
+
+p
+    |  #[+a(gh("thinc")) Thinc] is the machine learning library powering spaCy.
+    |  It's a practical toolkit for implementing models that follow the
+    |  #[+a("https://explosion.ai/blog/deep-learning-formula-nlp", true) "Embed, encode, attend, predict"]
+    |  architecture. It's designed to be easy to install, efficient for CPU
+    |  usage and optimised for NLP and deep learning with text – in particular,
+    |  hierarchically structured input and variable-length sequences.
+
++aside("How Thinc works")
+    |  To differentiate a function efficiently, you usually need to store
+    |  intermediate results, computed during the "forward pass", to reuse them
+    |  during the backward pass. Most libraries require the data passed through
+    |  the network to accumulate these intermediate result. In
+    |  #[+a(gh("thinc")) Thinc], a model
+    |  that computes #[code y = f(x)] is required to also
+    |  return a callback that computes #[code dx = f'(dy)]. Usually, the
+    |  callback is implemented as a closure, so the intermediate results can be
+    |  read from the enclosing scope.
+
+p
+    |  spaCy's built-in pipeline components can all be powered by any object
+    |  that follows Thinc's #[code Model] API. If a wrapper is not yet available
+    |  for the library you're using, you should create a
+    |  #[code thinc.neural.Model] subclass that implements a #[code begin_update]
+    |  method. You'll also want to implement #[code to_bytes], #[code from_bytes],
+    |  #[code to_disk] and #[code from_disk] methods, to save and load your
+    |  model.
+
++code("Thinc Model API").
+    class ThincModel(thinc.neural.Model):
+        def __init__(self, *args, **kwargs):
+            pass
+
+        def begin_update(self, X, drop=0.):
+            def backprop(dY, sgd=None):
+                return dX
+            return Y, backprop
+
+        def to_disk(self, path, **exclude):
+            return None
+
+        def from_disk(self, path, **exclude):
+            return self
+
+        def to_bytes(self, **exclude):
+            return bytes
+
+        def from_bytes(self, msgpacked_bytes, **exclude):
+            return self
+
+        def to_gpu(self, device_num):
+            return None
+
+        def to_cpu(self):
+            return None
+
+        def resize_output(self, new_size):
+            return None
+
+        def resize_input(self):
+            return None
+
+        @contextlib.contextmanager
+        def use_params(self, params):
+            return None
+
++table(["Method", "Description"])
+    +row
+        +cell #[code __init__]
+        +cell Initialise the model.
+
+    +row
+        +cell #[code begin_update]
+        +cell Return the output of the wrapped PyTorch model for the given input, along with a callback to handle the backward pass.
+
+    +row
+        +cell #[code to_disk]
+        +cell Save the model's weights to disk.
+
+    +row
+        +cell #[code from_disk]
+        +cell Read the model's weights from disk.
+
+    +row
+        +cell #[code to_bytes]
+        +cell Serialize the model's weights to bytes.
+
+    +row
+        +cell #[code from_bytes]
+        +cell Load the model's weights from bytes.
+
+    +row
+        +cell #[code to_gpu]
+        +cell
+            |  Ensure the model's weights are on the specified GPU device. If
+            |  already on that device, no action is taken.
+
+    +row
+        +cell #[code to_cpu]
+        +cell
+            |  Ensure the model's weights are on CPU. If already on CPU, no
+            |  action is taken.
+
+    +row
+        +cell #[code resize_output]
+        +cell
+            |  Resize the model such that the model's output vector has a new
+            |  size. If #[code new_size] is larger, weights corresponding to
+            |  the new output neurons are zero-initialized. If #[code new_size]
+            |  is smaller, neurons are dropped from the end of the vector.
+
+    +row
+        +cell #[code resize_input]
+        +cell
+            |  Resize the model such that the expects input vectors of a
+            |  different size. If #[code new_size] is larger, weights
+            |  corresponding to the new input neurons are zero-initialized. If
+            |  #[code new_size] is smaller, weights are dropped from the end of
+            |  the vector.
+
+    +row
+        +cell #[code use_params]
+        +cell
+            |  Use the given parameters, for the scope of the contextmanager.
+            |  At the end of the block, the weights are restored.

website/usage/deep-learning.jade

@@ -8,9 +8,9 @@ include ../_includes/_mixins
     +h(2, "pre-processing") Pre-processing text for deep learning
     include _deep-learning/_pre-processing
 
-+section("thinc")
-    +h(2, "thinc") spaCy and Thinc
-    include _deep-learning/_thinc
++section("wrapping")
+    +h(2, "wrapping") Wrapping models
+    include _deep-learning/_wrapping
 
 +section("tensorflow-keras")
     +h(2, "tensorflow-keras") Using spaCy with TensorFlow / Keras