Debug: show per-key manifest diffs in test_manifest_is_current

- setup.py: rename loop variable shadowing parameter (B020)
- _util.py: remove unused registry import (F401), use specific except clause (E722, B904)
- test_cli_app.py: use dict literals instead of dict() (C408)
- main.py: extract _try_static_group to reduce complexity (C901)

.github/workflows/cibuildwheel.yml

@@ -0,0 +1,24 @@
+name: Build
+
+on:
+  push:
+    tags:
+      # ytf did they invent their own syntax that's almost regex?
+      # ** matches 'zero or more of any character'
+      - 'release-v[0-9]+.[0-9]+.[0-9]+**'
+      - 'prerelease-v[0-9]+.[0-9]+.[0-9]+**'
+
+permissions: {}
+
+jobs:
+  build_wheels:
+    uses: explosion/gha-cibuildwheel/.github/workflows/cibuildwheel.yml@2c98f757f13d112cf73fcf4b627249f1fffb5aae  # main
+    permissions:
+      contents: write
+      actions: read
+    with:
+      wheel-name-pattern: "spacy-*.whl"
+      pure-python: false
+    secrets:
+      gh-token: ${{ secrets.GITHUB_TOKEN }}
+

.github/workflows/explosionbot.yml

@@ -6,6 +6,8 @@ on:
       - created
       - edited
 
+permissions: {}
+
 jobs:
   explosion-bot:
     if: github.repository_owner == 'explosion'
@@ -15,13 +17,15 @@ jobs:
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6
       - name: Install and run explosion-bot
         run: |
-          pip install git+https://${{ secrets.EXPLOSIONBOT_TOKEN }}@github.com/explosion/explosion-bot
+          git config --global url."https://x-access-token:${EXPLOSIONBOT_TOKEN}@github.com/".insteadOf "https://github.com/"
+          pip install git+https://github.com/explosion/explosion-bot
           python -m explosionbot
         env:
+          EXPLOSIONBOT_TOKEN: ${{ secrets.EXPLOSIONBOT_TOKEN }}
           INPUT_TOKEN: ${{ secrets.EXPLOSIONBOT_TOKEN }}
           INPUT_BK_TOKEN: ${{ secrets.BUILDKITE_SECRET }}
           ENABLED_COMMANDS: "test_gpu,test_slow,test_slow_gpu"

.github/workflows/issue-manager.yml

@@ -11,12 +11,16 @@ on:
     types:
       - labeled
 
+permissions: {}
+
 jobs:
   issue-manager:
+    permissions:
+      issues: write
     if: github.repository_owner == 'explosion'
     runs-on: ubuntu-latest
     steps:
-      - uses: tiangolo/issue-manager@0.4.0
+      - uses: tiangolo/issue-manager@4d1b7e05935a404dc8337d30bd23be46be8bb8e5  # 0.4.0
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           config: >

.github/workflows/lock.yml

@@ -16,7 +16,7 @@ jobs:
     if: github.repository_owner == 'explosion'
     runs-on: ubuntu-latest
     steps:
-      - uses: dessant/lock-threads@v5
+      - uses: dessant/lock-threads@1bf7ec25051fe7c00bdd17e6a7cf3d7bfb7dc771  # v5
         with:
           process-only: 'issues'
           issue-inactive-days: '30'

.github/workflows/publish_pypi.yml

@@ -0,0 +1,31 @@
+# The cibuildwheel action triggers on creation of a release, this
+# triggers on publication.
+# The expected workflow is to create a draft release and let the wheels
+# upload, and then hit 'publish', which uploads to PyPi.
+
+on:
+  release:
+    types:
+      - published
+
+permissions: {}
+
+jobs:
+  upload_pypi:
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/spacy
+    permissions:
+      id-token: write
+      contents: read
+    if: github.event_name == 'release' && github.event.action == 'published'
+    # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this)
+    # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - uses: robinraju/release-downloader@daf26c55d821e836577a15f77d86ddc078948b05  # v1
+        with:
+          tag: ${{ github.event.release.tag_name }}
+          fileName: '*'
+          out-file-path: 'dist'
+      - uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/spacy_universe_alert.yml

@@ -5,21 +5,16 @@ on:
     paths:
       - "website/meta/universe.json"
 
+permissions: {}
+
 jobs:
   build:
     if: github.repository_owner == 'explosion'
     runs-on: ubuntu-latest
 
     steps:
-      - name: Dump GitHub context
-        env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
-          PR_NUMBER: ${{github.event.number}}
-        run: |
-          echo "$GITHUB_CONTEXT"
-
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6
         with:
           python-version: '3.10'
       - name: Install Bernadette app dependency and send an alert

.github/workflows/tests.yml

@@ -2,6 +2,8 @@ name: tests
 
 on:
   push:
+    tags-ignore:
+      - '**'
     branches-ignore:
       - "spacy.io"
       - "nightly.spacy.io"
@@ -10,7 +12,6 @@ on:
       - "*.md"
       - "*.mdx"
       - "website/**"
-      - ".github/workflows/**"
   pull_request:
     types: [opened, synchronize, reopened, edited]
     paths-ignore:
@@ -18,6 +19,8 @@ on:
       - "*.mdx"
       - "website/**"
 
+permissions: {}
+
 jobs:
   validate:
     name: Validate
@@ -25,55 +28,38 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
 
       - name: Configure Python version
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6
         with:
-          python-version: "3.9"
+          python-version: "3.10"
 
-      - name: black
+      - name: ruff format
         run: |
-          python -m pip install black -c requirements.txt
-          python -m black spacy --check
-      - name: isort
+          python -m pip install ruff -c requirements.txt
+          python -m ruff format spacy --check
+      - name: ruff isort
         run: |
-          python -m pip install isort -c requirements.txt
-          python -m isort spacy --check
-      - name: flake8
-        run: |
-          python -m pip install flake8==5.0.4
-          python -m flake8 spacy --count --select=E901,E999,F821,F822,F823,W605 --show-source --statistics
-      - name: cython-lint
-        run: |
-          python -m pip install cython-lint -c requirements.txt
-          # E501: line too log, W291: trailing whitespace, E266: too many leading '#' for block comment
-          cython-lint spacy --ignore E501,W291,E266
+          python -m ruff check spacy --select I
 
   tests:
     name: Test
     needs: Validate
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python_version: ["3.12"]
-        include:
-          - os: ubuntu-20.04
-            python_version: "3.9"
-          - os: windows-latest
-            python_version: "3.10"
-          - os: macos-latest
-            python_version: "3.11"
+        python_version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
 
     runs-on: ${{ matrix.os }}
 
     steps:
       - name: Check out repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
 
       - name: Configure Python version
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6
         with:
           python-version: ${{ matrix.python_version }}
 
@@ -89,6 +75,7 @@ jobs:
       - name: Run mypy
         run: |
           python -m mypy spacy
+        if: matrix.python_version != '3.7'
 
       - name: Delete source directory and .egg-info
         run: |
@@ -108,24 +95,24 @@ jobs:
         shell: bash
 
       - name: Test import
-        run: python -W error -c "import spacy"
+        run: python -W error -W 'ignore:Core Pydantic V1:UserWarning:pydantic' -c "import spacy"
 
-      #      - name: "Test download CLI"
-      #        run: |
-      #          python -m spacy download ca_core_news_sm
-      #          python -m spacy download ca_core_news_md
-      #          python -c "import spacy; nlp=spacy.load('ca_core_news_sm'); doc=nlp('test')"
-      #        if: matrix.python_version == '3.9'
-      #
-      #      - name: "Test download_url in info CLI"
-      #        run: |
-      #          python -W error -m spacy info ca_core_news_sm | grep -q download_url
-      #        if: matrix.python_version == '3.9'
-      #
-      #      - name: "Test no warnings on load (#11713)"
-      #        run: |
-      #          python -W error -c "import ca_core_news_sm; nlp = ca_core_news_sm.load(); doc=nlp('test')"
-      #        if: matrix.python_version == '3.9'
+      - name: "Test download CLI"
+        run: |
+          python -m spacy download ca_core_news_sm
+          python -m spacy download ca_core_news_md
+          python -c "import spacy; nlp=spacy.load('ca_core_news_sm'); doc=nlp('test')"
+        if: matrix.python_version == '3.9'
+
+      - name: "Test download_url in info CLI"
+        run: |
+          python -W error -m spacy info ca_core_news_sm | grep -q download_url
+        if: matrix.python_version == '3.9'
+
+      - name: "Test no warnings on load (#11713)"
+        run: |
+          python -W error -c "import ca_core_news_sm; nlp = ca_core_news_sm.load(); doc=nlp('test')"
+        if: matrix.python_version == '3.9'
 
       - name: "Test convert CLI"
         run: |
@@ -149,17 +136,19 @@ jobs:
           python -m spacy train ner.cfg --paths.train ner-token-per-line-conll2003.spacy --paths.dev ner-token-per-line-conll2003.spacy --training.max_steps 10 --gpu-id -1
         if: matrix.python_version == '3.9'
 
-      #      - name: "Test assemble CLI"
-      #        run: |
-      #          python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_sm'}; config.to_disk('ner_source_sm.cfg')"
-      #          PYTHONWARNINGS="error,ignore::DeprecationWarning" python -m spacy assemble ner_source_sm.cfg output_dir
-      #        if: matrix.python_version == '3.9'
-      #
-      #      - name: "Test assemble CLI vectors warning"
-      #        run: |
-      #          python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_md'}; config.to_disk('ner_source_md.cfg')"
-      #          python -m spacy assemble ner_source_md.cfg output_dir 2>&1 | grep -q W113
-      #        if: matrix.python_version == '3.9'
+      - name: "Test assemble CLI"
+        run: |
+          python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_sm'}; config.to_disk('ner_source_sm.cfg')"
+          python -m spacy assemble ner_source_sm.cfg output_dir
+        env:
+          PYTHONWARNINGS: "error,ignore::DeprecationWarning" 
+        if: matrix.python_version == '3.9'
+
+      - name: "Test assemble CLI vectors warning"
+        run: |
+          python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_md'}; config.to_disk('ner_source_md.cfg')"
+          python -m spacy assemble ner_source_md.cfg output_dir 2>&1 | grep -q W113
+        if: matrix.python_version == '3.9'
 
       - name: "Install test requirements"
         run: |
@@ -167,5 +156,11 @@ jobs:
 
       - name: "Run CPU tests"
         run: |
-          python -m pytest --pyargs spacy -W error
-        if: matrix.python_version == '3.11'
+          python -m pytest --pyargs spacy -W error -W 'ignore:Core Pydantic V1:UserWarning:pydantic'
+        if: "!(startsWith(matrix.os, 'macos') && matrix.python_version == '3.11')"
+
+      - name: "Run CPU tests with thinc-apple-ops"
+        run: |
+          python -m pip install 'spacy[apple]'
+          python -m pytest --pyargs spacy
+        if: startsWith(matrix.os, 'macos') && matrix.python_version == '3.11'

.github/workflows/universe_validation.yml

@@ -13,6 +13,8 @@ on:
     paths:
       - "website/meta/universe.json"
 
+permissions: {}
+
 jobs:
   validate:
     name: Validate
@@ -20,12 +22,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
 
       - name: Configure Python version
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6
         with:
-          python-version: "3.9"
+          python-version: "3.7"
 
       - name: Validate website/meta/universe.json
         run: |

.pre-commit-config.yaml

@@ -1,13 +1,7 @@
 repos:
-  - repo: https://github.com/ambv/black
-    rev: 22.3.0
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.0
     hooks:
-      - id: black
-        language_version: python3.9
-        additional_dependencies: ["click==8.0.4"]
-  - repo: https://github.com/pycqa/flake8
-    rev: 5.0.4
-    hooks:
-      - id: flake8
-        args:
-          - "--config=setup.cfg"
+    - id: ruff
+      args: ['--fix']
+    - id: ruff-format

CONTRIBUTING.md

@@ -35,7 +35,7 @@ so that more people can benefit from it.
 
 When opening an issue, use a **descriptive title** and include your
 **environment** (operating system, Python version, spaCy version). Our
-[issue template](https://github.com/explosion/spaCy/issues/new) helps you
+[issue templates](https://github.com/explosion/spaCy/issues/new/choose) help you
 remember the most important details to include. If you've discovered a bug, you
 can also submit a [regression test](#fixing-bugs) straight away. When you're
 opening an issue to report the bug, simply refer to your pull request in the
@@ -276,7 +276,7 @@ except:  # noqa: E722
 
 ### Python conventions
 
-All Python code must be written **compatible with Python 3.9+**. More detailed
+All Python code must be written **compatible with Python 3.6+**. More detailed
 code conventions can be found in the [developer docs](https://github.com/explosion/spaCy/blob/master/extra/DEVELOPER_DOCS/Code%20Conventions.md).
 
 #### I/O and handling paths
@@ -449,8 +449,8 @@ and plugins in spaCy v3.0, and we can't wait to see what you build with it!
   [`spacy`](https://github.com/topics/spacy?o=desc&s=stars) and
   [`spacy-extensions`](https://github.com/topics/spacy-extension?o=desc&s=stars)
   to make it easier to find. Those are also the topics we're linking to from the
-  spaCy website. If you're sharing your project on Twitter, feel free to tag
-  [@spacy_io](https://twitter.com/spacy_io) so we can check it out.
+  spaCy website. If you're sharing your project on X, feel free to tag
+  [@spacy_io](https://x.com/spacy_io) so we can check it out.
 
 - Once your extension is published, you can open a
   [PR](https://github.com/explosion/spaCy/pulls) to suggest it for the

MANIFEST.in

@@ -1,8 +1,10 @@
 recursive-include spacy *.pyi *.pyx *.pxd *.txt *.cfg *.jinja *.toml *.hh
+recursive-include spacy_cli *.json
 include LICENSE
 include README.md
 include pyproject.toml
 include spacy/py.typed
 recursive-include spacy/cli *.yml
+recursive-include spacy/tests *.json
 recursive-include licenses *
 recursive-exclude spacy *.cpp

Makefile

@@ -5,7 +5,7 @@ override SPACY_EXTRAS = spacy-lookups-data==1.0.3
 endif
 
 ifndef PYVER
-override PYVER = 3.9
+override PYVER = 3.8
 endif
 
 VENV := ./env$(PYVER)

README.md

@@ -16,7 +16,7 @@ model packaging, deployment and workflow management. spaCy is commercial
 open-source software, released under the
 [MIT license](https://github.com/explosion/spaCy/blob/master/LICENSE).
 
-💫 **Version 3.7 out now!**
+💫 **Version 3.8 out now!**
 [Check out the release notes here.](https://github.com/explosion/spaCy/releases)
 
 [![tests](https://github.com/explosion/spaCy/actions/workflows/tests.yml/badge.svg)](https://github.com/explosion/spaCy/actions/workflows/tests.yml)
@@ -28,29 +28,29 @@ open-source software, released under the
 <br />
 [![PyPi downloads](https://static.pepy.tech/personalized-badge/spacy?period=total&units=international_system&left_color=grey&right_color=orange&left_text=pip%20downloads)](https://pypi.org/project/spacy/)
 [![Conda downloads](https://img.shields.io/conda/dn/conda-forge/spacy?label=conda%20downloads)](https://anaconda.org/conda-forge/spacy)
-[![spaCy on Twitter](https://img.shields.io/twitter/follow/spacy_io.svg?style=social&label=Follow)](https://twitter.com/spacy_io)
 
 ## 📖 Documentation
 
-| Documentation                                                                                                                                                                                   |                                                                                                                                                                                                                                                                                                       |
-| ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| ⭐️ **[spaCy 101]**                                                                                                                                                                             | New to spaCy? Here's everything you need to know!                                                                                                                                                                                                                                                     |
-| 📚 **[Usage Guides]**                                                                                                                                                                           | How to use spaCy and its features.                                                                                                                                                                                                                                                                    |
-| 🚀 **[New in v3.0]**                                                                                                                                                                            | New features, backwards incompatibilities and migration guide.                                                                                                                                                                                                                                        |
-| 🪐 **[Project Templates]**                                                                                                                                                                      | End-to-end workflows you can clone, modify and run.                                                                                                                                                                                                                                                   |
-| 🎛 **[API Reference]**                                                                                                                                                                           | The detailed reference for spaCy's API.                                                                                                                                                                                                                                                               |
-| ⏩ **[GPU Processing]**                                                                                                                                                                         | Use spaCy with CUDA-compatible GPU processing.                                                                                                                                                                                                                                                        |
-| 📦 **[Models]**                                                                                                                                                                                 | Download trained pipelines for spaCy.                                                                                                                                                                                                                                                                 |
-| 🦙 **[Large Language Models]**                                                                                                                                                                  | Integrate LLMs into spaCy pipelines.                                                                                                                                                                                                                                                                  |
-| 🌌 **[Universe]**                                                                                                                                                                               | Plugins, extensions, demos and books from the spaCy ecosystem.                                                                                                                                                                                                                                        |
-| ⚙️ **[spaCy VS Code Extension]**                                                                                                                                                                | Additional tooling and features for working with spaCy's config files.                                                                                                                                                                                                                                |
-| 👩‍🏫 **[Online Course]**                                                                                                                                                                          | Learn spaCy in this free and interactive online course.                                                                                                                                                                                                                                               |
-| 📰 **[Blog]**                                                                                                                                                                                   | Read about current spaCy and Prodigy development, releases, talks and more from Explosion.                                                                                                                                                                                                            |
-| 📺 **[Videos]**                                                                                                                                                                                 | Our YouTube channel with video tutorials, talks and more.                                                                                                                                                                                                                                             |
-| 🛠 **[Changelog]**                                                                                                                                                                               | Changes and version history.                                                                                                                                                                                                                                                                          |
-| 💝 **[Contribute]**                                                                                                                                                                             | How to contribute to the spaCy project and code base.                                                                                                                                                                                                                                                 |
-| 👕 **[Swag]**                                                                                                                                                                                   | Support us and our work with unique, custom-designed swag!                                                                                                                                                                                                                                            |
-| <a href="https://explosion.ai/tailored-solutions"><img src="https://github.com/explosion/spaCy/assets/13643239/36d2a42e-98c0-4599-90e1-788ef75181be" width="150" alt="Tailored Solutions"/></a> | Custom NLP consulting, implementation and strategic advice by spaCy’s core development team. Streamlined, production-ready, predictable and maintainable. Send us an email or take our 5-minute questionnaire, and well'be in touch! **[Learn more &rarr;](https://explosion.ai/tailored-solutions)** |
+| Documentation                                                                                                                                                                                                             |                                                                                                                                                                                                                                                                                                                                              |
+| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| ⭐️ **[spaCy 101]**                                                                                                                                                                                                       | New to spaCy? Here's everything you need to know!                                                                                                                                                                                                                                                                                            |
+| 📚 **[Usage Guides]**                                                                                                                                                                                                     | How to use spaCy and its features.                                                                                                                                                                                                                                                                                                           |
+| 🚀 **[New in v3.0]**                                                                                                                                                                                                      | New features, backwards incompatibilities and migration guide.                                                                                                                                                                                                                                                                               |
+| 🪐 **[Project Templates]**                                                                                                                                                                                                | End-to-end workflows you can clone, modify and run.                                                                                                                                                                                                                                                                                          |
+| 🎛 **[API Reference]**                                                                                                                                                                                                     | The detailed reference for spaCy's API.                                                                                                                                                                                                                                                                                                      |
+| ⏩ **[GPU Processing]**                                                                                                                                                                                                    | Use spaCy with CUDA-compatible GPU processing.                                                                                                                                                                                                                                                                                               |
+| 📦 **[Models]**                                                                                                                                                                                                           | Download trained pipelines for spaCy.                                                                                                                                                                                                                                                                                                        |
+| 🦙 **[Large Language Models]**                                                                                                                                                                                            | Integrate LLMs into spaCy pipelines.                                                                                                                                                                                                                                                                                                        |
+| 🌌 **[Universe]**                                                                                                                                                                                                         | Plugins, extensions, demos and books from the spaCy ecosystem.                                                                                                                                                                                                                                                                               |
+| ⚙️ **[spaCy VS Code Extension]**                                                                                                                                                                                          | Additional tooling and features for working with spaCy's config files.                                                                                                                                                                                                                                                                       |
+| 👩‍🏫 **[Online Course]**                                                                                                                                                                                                    | Learn spaCy in this free and interactive online course.                                                                                                                                                                                                                                                                                      |
+| 📰 **[Blog]**                                                                                                                                                                                                             | Read about current spaCy and Prodigy development, releases, talks and more from Explosion.                                                                                                                                                                                                                 |
+| 📺 **[Videos]**                                                                                                                                                                                                           | Our YouTube channel with video tutorials, talks and more.                                                                                                                                                                                                                                                                                    |
+| 🔴 **[Live Stream]**                                                                                                                                                                                                       | Join Matt as he works on spaCy and chat about NLP, live every week.                                                                                                                                                                                                                                                                         |
+| 🛠 **[Changelog]**                                                                                                                                                                                                         | Changes and version history.                                                                                                                                                                                                                                                                                                                 |
+| 💝 **[Contribute]**                                                                                                                                                                                                       | How to contribute to the spaCy project and code base.                                                                                                                                                                                                                                                                                        |
+| 👕 **[Swag]**                                                                                                                                                                                                             | Support us and our work with unique, custom-designed swag!                                                                                                                                                                                                                                                                                   |
+| <a href="https://explosion.ai/tailored-solutions"><img src="https://github.com/explosion/spaCy/assets/13643239/36d2a42e-98c0-4599-90e1-788ef75181be" width="150" alt="Tailored Solutions"/></a> | Custom NLP consulting, implementation and strategic advice by spaCy’s core development team. Streamlined, production-ready, predictable and maintainable. Send us an email or take our 5-minute questionnaire, and well'be in touch! **[Learn more &rarr;](https://explosion.ai/tailored-solutions)**                 |
 
 [spacy 101]: https://spacy.io/usage/spacy-101
 [new in v3.0]: https://spacy.io/usage/v3
@@ -62,6 +62,7 @@ open-source software, released under the
 [universe]: https://spacy.io/universe
 [spacy vs code extension]: https://github.com/explosion/spacy-vscode
 [videos]: https://www.youtube.com/c/ExplosionAI
+[live stream]: https://www.youtube.com/playlist?list=PLBmcuObd5An5_iAxNYLJa_xWmNzsYce8c
 [online course]: https://course.spacy.io
 [blog]: https://explosion.ai
 [project templates]: https://github.com/explosion/projects
@@ -79,13 +80,14 @@ more people can benefit from it.
 | Type                            | Platforms                               |
 | ------------------------------- | --------------------------------------- |
 | 🚨 **Bug Reports**              | [GitHub Issue Tracker]                  |
-| 🎁 **Feature Requests & Ideas** | [GitHub Discussions]                    |
+| 🎁 **Feature Requests & Ideas** | [GitHub Discussions] · [Live Stream]    |
 | 👩‍💻 **Usage Questions**          | [GitHub Discussions] · [Stack Overflow] |
-| 🗯 **General Discussion**        | [GitHub Discussions]                    |
+| 🗯 **General Discussion**        | [GitHub Discussions] · [Live Stream]   |
 
 [github issue tracker]: https://github.com/explosion/spaCy/issues
 [github discussions]: https://github.com/explosion/spaCy/discussions
 [stack overflow]: https://stackoverflow.com/questions/tagged/spacy
+[live stream]: https://www.youtube.com/playlist?list=PLBmcuObd5An5_iAxNYLJa_xWmNzsYce8c
 
 ## Features
 
@@ -115,7 +117,7 @@ For detailed installation instructions, see the
 
 - **Operating system**: macOS / OS X · Linux · Windows (Cygwin, MinGW, Visual
   Studio)
-- **Python version**: Python 3.9+ (only 64 bit)
+- **Python version**: Python >=3.7, <3.13 (only 64 bit)
 - **Package managers**: [pip] · [conda] (via `conda-forge`)
 
 [pip]: https://pypi.org/project/spacy/

bin/release.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Insist repository is clean
+git diff-index --quiet HEAD
+
+version=$(grep "__version__ = " spacy/about.py)
+version=${version/__version__ = }
+version=${version/\'/}
+version=${version/\'/}
+version=${version/\"/}
+version=${version/\"/}
+
+echo "Pushing release-v"$version
+
+git tag -d release-v$version || true
+git push origin :release-v$version || true
+git tag release-v$version
+git push origin release-v$version

build-constraints.txt

@@ -1,2 +1,2 @@
 # build version constraints for use with wheelwright
-numpy>=2.0.0; python_version>='3.9'
+numpy>=2.0.0,<3.0.0

extra/DEVELOPER_DOCS/Satellite Packages.md

@@ -16,7 +16,7 @@ These packages are always pulled in when you install spaCy. Most of them are dir
 - [preshed](https://github.com/explosion/preshed): A Cython library for low-level data structures like hash maps, used for memory efficient data storage.
 - [cython-blis](https://github.com/explosion/cython-blis): Fast matrix multiplication using BLIS without depending on system libraries. Required by Thinc, rather than spaCy directly.
 - [murmurhash](https://github.com/explosion/murmurhash): A wrapper library for a C++ murmurhash implementation, used for string IDs in spaCy and preshed.
-- [cymem](https://github.com/explosion/cymem): A small library for RAII-style memory management in Cython.
+- [cymem](https://github.com/explosion/cymem): A small library for RAII-style memory management in Cython. 
 
 ## Optional Extensions for spaCy
 
@@ -25,12 +25,13 @@ These are repos that can be used by spaCy but aren't part of a default insta
 - [spacy-transformers](https://github.com/explosion/spacy-transformers): A wrapper for the [HuggingFace Transformers](https://huggingface.co/docs/transformers/index) library, this handles the extensive conversion necessary to coordinate spaCy's powerful `Doc` representation, training pipeline, and the Transformer embeddings. When released, this was known as `spacy-pytorch-transformers`, but it changed to the current name when HuggingFace update the name of their library as well.
 - [spacy-huggingface-hub](https://github.com/explosion/spacy-huggingface-hub): This package has a CLI script for uploading a packaged spaCy pipeline (created with `spacy package`) to the [Hugging Face Hub](https://huggingface.co/models).
 - [spacy-alignments](https://github.com/explosion/spacy-alignments): A wrapper for the tokenizations library (mentioned below) with a modified build system to simplify cross-platform wheel creation. Used in spacy-transformers for aligning spaCy and HuggingFace tokenizations.
-- [spacy-experimental](https://github.com/explosion/spacy-experimental): Experimental components that are not quite ready for inclusion in the main spaCy library. Usually there are unresolved questions around their APIs, so the experimental library allows us to expose them to the community for feedback before fully integrating them.
+- [spacy-experimental](https://github.com/explosion/spacy-experimental): Experimental components that are not quite ready for inclusion in the main spaCy library. Usually there are unresolved questions around their APIs, so the experimental library allows us to expose them to the community for feedback before fully integrating them. 
 - [spacy-lookups-data](https://github.com/explosion/spacy-lookups-data): A repository of linguistic data, such as lemmas, that takes up a lot of disk space. Originally created to reduce the size of the spaCy core library. This is mainly useful if you want the data included but aren't using a pretrained pipeline; for the affected languages, the relevant data is included in pretrained pipelines directly.
 - [coreferee](https://github.com/explosion/coreferee): Coreference resolution for English, French, German and Polish, optimised for limited training data and easily extensible for further languages. Used as a spaCy pipeline component.
-- [spacy-stanza](https://github.com/explosion/spacy-stanza): This is a wrapper that allows the use of Stanford's Stanza library in spaCy.
+- [spacy-stanza](https://github.com/explosion/spacy-stanza): This is a wrapper that allows the use of Stanford's Stanza library in spaCy. 
 - [spacy-streamlit](https://github.com/explosion/spacy-streamlit): A wrapper for the Streamlit dashboard building library to help with integrating [displaCy](https://spacy.io/api/top-level/#displacy).
 - [spacymoji](https://github.com/explosion/spacymoji): A library to add extra support for emoji to spaCy, such as including character names.
+- [thinc-apple-ops](https://github.com/explosion/thinc-apple-ops): A special backend for OSX that uses Apple's native libraries for improved performance.
 - [os-signpost](https://github.com/explosion/os-signpost): A Python package that allows you to use the `OSSignposter` API in OSX for performance analysis.
 - [spacy-ray](https://github.com/explosion/spacy-ray): A wrapper to integrate spaCy with Ray, a distributed training framework. Currently a work in progress.
 
@@ -78,3 +79,4 @@ Repos that don't fit in any of the above categories.
 - [tokenizations](https://github.com/explosion/tokenizations): A library originally by Yohei Tamura to align strings with tolerance to some variations in features like case and diacritics, used for aligning tokens and wordpieces. Adopted and maintained by Explosion, but usually spacy-alignments is used instead.
 - [conll-2012](https://github.com/explosion/conll-2012): A repo to hold some slightly cleaned up versions of the official scripts for the CoNLL 2012 shared task involving coreference resolution. Used in the coref project.
 - [fastapi-explosion-extras](https://github.com/explosion/fastapi-explosion-extras): Some small tweaks to FastAPI used at Explosion.
+

lint.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Local lint script matching the CI Validate job + mypy type checks.
+# Fixes formatting and import sorting in-place, then re-verifies in
+# check mode to catch any conflicts between the two, and runs mypy.
+set -euo pipefail
+
+err=0
+
+echo "==> ruff format (auto-fixing)"
+python -m ruff format spacy
+
+echo "==> ruff isort (auto-fixing)"
+python -m ruff check spacy --select I --fix
+
+echo "==> ruff format (verify)"
+if ! python -m ruff format spacy --check; then
+    echo "FAIL: isort fix broke formatting"
+    err=1
+fi
+
+echo "==> ruff isort (verify)"
+if ! python -m ruff check spacy --select I; then
+    echo "FAIL: format fix broke import sorting"
+    err=1
+fi
+
+echo "==> mypy"
+if ! python -m mypy spacy; then
+    err=1
+fi
+
+if [ "$err" -ne 0 ]; then
+    echo "FAIL: see errors above"
+    exit 1
+fi
+
+echo "OK: all checks passed"

pyproject.toml

@@ -1,15 +1,74 @@
 [build-system]
 requires = [
     "setuptools",
-    "cython>=0.25,<3.0",
+    "cython>=3.0,<4.0",
     "cymem>=2.0.2,<2.1.0",
     "preshed>=3.0.2,<3.1.0",
     "murmurhash>=0.28.0,<1.1.0",
-    "thinc>=9.1.0,<9.2.0",
-    "numpy>=2.0.0; python_version < '3.9'",
-    "numpy>=2.0.0; python_version >= '3.9'",
+    "thinc>=8.3.12,<8.4.0",
+    "numpy>=2.0.0,<3.0.0"
 ]
 build-backend = "setuptools.build_meta"
 
-[tool.isort]
-profile = "black"
+[tool.cibuildwheel]
+build = "*"
+skip = "cp39* *-win32 *i686* cp3??t-* *cp310-win_arm64"
+test-skip = ""
+
+archs = ["native"]
+
+build-frontend = "default"
+config-settings = {}
+dependency-versions = "pinned"
+environment = { PIP_CONSTRAINT = "build-constraints.txt" }
+
+environment-pass = []
+build-verbosity = 0
+
+before-all = "curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain stable"
+before-build = "pip install -r requirements.txt && python setup.py clean"
+repair-wheel-command = ""
+
+test-command = ""
+before-test = ""
+test-requires = []
+test-extras = []
+
+container-engine = "docker"
+
+manylinux-x86_64-image = "manylinux2014"
+manylinux-i686-image = "manylinux2014"
+manylinux-aarch64-image = "manylinux2014"
+manylinux-ppc64le-image = "manylinux2014"
+manylinux-s390x-image = "manylinux2014"
+manylinux-pypy_x86_64-image = "manylinux2014"
+manylinux-pypy_i686-image = "manylinux2014"
+manylinux-pypy_aarch64-image = "manylinux2014"
+
+musllinux-x86_64-image = "musllinux_1_2"
+musllinux-i686-image = "musllinux_1_2"
+musllinux-aarch64-image = "musllinux_1_2"
+musllinux-ppc64le-image = "musllinux_1_2"
+musllinux-s390x-image = "musllinux_1_2"
+
+[tool.cibuildwheel.linux]
+repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}"
+
+[tool.cibuildwheel.macos]
+repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}"
+
+[tool.cibuildwheel.windows]
+
+[tool.cibuildwheel.pyodide]
+
+
+[tool.ruff]
+line-length = 88
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "C", "B", "B9"]
+ignore = ["E203", "E266", "E501", "E731", "E741", "F541"]
+
+[tool.ruff.lint.isort]
+combine-as-imports = true
+split-on-trailing-comma = true

requirements.txt

@@ -1,40 +1,37 @@
 # Our libraries
-spacy-legacy>=4.0.0.dev1,<4.1.0
+spacy-legacy>=3.0.11,<3.1.0
 spacy-loggers>=1.0.0,<2.0.0
 cymem>=2.0.2,<2.1.0
 preshed>=3.0.2,<3.1.0
-thinc>=9.1.0,<9.2.0
-ml_datasets>=0.2.0,<0.3.0
+thinc>=8.3.12,<8.4.0
+ml_datasets>=0.2.1,<0.3.0
 murmurhash>=0.28.0,<1.1.0
 wasabi>=0.9.1,<1.2.0
-srsly>=2.4.3,<3.0.0
+srsly>=2.5.3,<3.0.0
 catalogue>=2.0.6,<2.1.0
 typer>=0.3.0,<1.0.0
-weasel>=0.1.0,<0.5.0
+weasel>=1.0.0,<2.0.0
 # Third party dependencies
-numpy>=2.0.0,<3.0.0; python_version < "3.9"
-numpy>=2.0.0,<3.0.0; python_version >= "3.9"
+numpy>=2.0.0,<3.0.0
 requests>=2.13.0,<3.0.0
 tqdm>=4.38.0,<5.0.0
-pydantic>=1.7.4,!=1.8,!=1.8.1,<3.0.0
+pydantic>=2.0.0,<3.0.0
 jinja2
-langcodes>=3.2.0,<4.0.0
 # Official Python utilities
 setuptools
 packaging>=20.0
 # Development dependencies
 pre-commit>=2.13.0
-cython>=0.25,<3.0
+cython>=3.0,<4.0
 pytest>=5.2.0,!=7.1.0
 pytest-timeout>=1.3.0,<2.0.0
 mock>=2.0.0,<3.0.0
-flake8>=3.8.0,<6.0.0
 hypothesis>=3.27.0,<7.0.0
-mypy>=1.5.0,<1.6.0; platform_machine != "aarch64"
+mypy>=1.5.0,<1.6.0; platform_machine != "aarch64" and python_version >= "3.8"
 types-mock>=0.1.1
 types-setuptools>=57.0.0
 types-requests
 types-setuptools>=57.0.0
-black==22.3.0
+ruff>=0.9.0
 cython-lint>=0.15.0
-isort>=5.0,<6.0
+confection>=1.1.0,<2.0.0

setup.cfg

@@ -21,6 +21,8 @@ classifiers =
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
     Programming Language :: Python :: 3.12
+    Programming Language :: Python :: 3.13
+    Programming Language :: Python :: 3.14
     Topic :: Scientific/Engineering
 project_urls =
     Release notes = https://github.com/explosion/spaCy/releases
@@ -29,35 +31,46 @@ project_urls =
 [options]
 zip_safe = false
 include_package_data = true
-python_requires = >=3.9
+python_requires = >=3.9,<3.15
+# NOTE: This section is superseded by pyproject.toml and will be removed in
+# spaCy v4
+setup_requires =
+    cython>=3.0,<4.0
+    numpy>=2.0.0,<3.0.0; python_version < "3.9"
+    numpy>=2.0.0,<3.0.0; python_version >= "3.9"
+    # We also need our Cython packages here to compile against
+    cymem>=2.0.2,<2.1.0
+    preshed>=3.0.2,<3.1.0
+    murmurhash>=0.28.0,<1.1.0
+    thinc>=8.3.12,<8.4.0
 install_requires =
     # Our libraries
-    spacy-legacy>=4.0.0.dev1,<4.1.0
+    spacy-legacy>=3.0.11,<3.1.0
     spacy-loggers>=1.0.0,<2.0.0
     murmurhash>=0.28.0,<1.1.0
     cymem>=2.0.2,<2.1.0
     preshed>=3.0.2,<3.1.0
-    thinc>=9.1.0,<9.2.0
+    thinc>=8.3.12,<8.4.0
     wasabi>=0.9.1,<1.2.0
-    srsly>=2.4.3,<3.0.0
+    srsly>=2.5.3,<3.0.0
     catalogue>=2.0.6,<2.1.0
-    weasel>=0.1.0,<0.5.0
+    weasel>=1.0.0,<2.0.0
+    confection>=1.1.0,<2.0.0
     # Third-party dependencies
     typer>=0.3.0,<1.0.0
     tqdm>=4.38.0,<5.0.0
-    numpy>=2.0.0,<3.0.0; python_version < "3.9"
-    numpy>=2.0.0,<3.0.0; python_version >= "3.9"
+    numpy>=1.15.0; python_version < "3.9"
+    numpy>=1.19.0; python_version >= "3.9"
     requests>=2.13.0,<3.0.0
-    pydantic>=1.7.4,!=1.8,!=1.8.1,<3.0.0
+    pydantic>=2.0.0,<3.0.0
     jinja2
     # Official Python utilities
     setuptools
     packaging>=20.0
-    langcodes>=3.2.0,<4.0.0
 
 [options.entry_points]
 console_scripts =
-    spacy = spacy.cli:setup_cli
+    spacy = spacy_cli.main:main
 
 [options.extras_require]
 lookups =
@@ -65,49 +78,51 @@ lookups =
 transformers =
     spacy_transformers>=1.1.2,<1.4.0
 cuda =
-    cupy>=5.0.0b4,<14.0.0
+    cupy>=5.0.0b4,<13.0.0
 cuda80 =
-    cupy-cuda80>=5.0.0b4,<14.0.0
+    cupy-cuda80>=5.0.0b4,<13.0.0
 cuda90 =
-    cupy-cuda90>=5.0.0b4,<14.0.0
+    cupy-cuda90>=5.0.0b4,<13.0.0
 cuda91 =
-    cupy-cuda91>=5.0.0b4,<14.0.0
+    cupy-cuda91>=5.0.0b4,<13.0.0
 cuda92 =
-    cupy-cuda92>=5.0.0b4,<14.0.0
+    cupy-cuda92>=5.0.0b4,<13.0.0
 cuda100 =
-    cupy-cuda100>=5.0.0b4,<14.0.0
+    cupy-cuda100>=5.0.0b4,<13.0.0
 cuda101 =
-    cupy-cuda101>=5.0.0b4,<14.0.0
+    cupy-cuda101>=5.0.0b4,<13.0.0
 cuda102 =
-    cupy-cuda102>=5.0.0b4,<14.0.0
+    cupy-cuda102>=5.0.0b4,<13.0.0
 cuda110 =
-    cupy-cuda110>=5.0.0b4,<14.0.0
+    cupy-cuda110>=5.0.0b4,<13.0.0
 cuda111 =
-    cupy-cuda111>=5.0.0b4,<14.0.0
+    cupy-cuda111>=5.0.0b4,<13.0.0
 cuda112 =
-    cupy-cuda112>=5.0.0b4,<14.0.0
+    cupy-cuda112>=5.0.0b4,<13.0.0
 cuda113 =
-    cupy-cuda113>=5.0.0b4,<14.0.0
+    cupy-cuda113>=5.0.0b4,<13.0.0
 cuda114 =
-    cupy-cuda114>=5.0.0b4,<14.0.0
+    cupy-cuda114>=5.0.0b4,<13.0.0
 cuda115 =
-    cupy-cuda115>=5.0.0b4,<14.0.0
+    cupy-cuda115>=5.0.0b4,<13.0.0
 cuda116 =
-    cupy-cuda116>=5.0.0b4,<14.0.0
+    cupy-cuda116>=5.0.0b4,<13.0.0
 cuda117 =
-    cupy-cuda117>=5.0.0b4,<14.0.0
+    cupy-cuda117>=5.0.0b4,<13.0.0
 cuda11x =
-    cupy-cuda11x>=12.0.0,<14.0.0
+    cupy-cuda11x>=11.0.0,<13.0.0
 cuda12x =
-    cupy-cuda12x>=11.5.0,<14.0.0
+    cupy-cuda12x>=11.5.0,<13.0.0
 cuda-autodetect =
-    cupy-wheel>=11.0.0,<14.0.0
+    cupy-wheel>=11.0.0,<13.0.0
+apple =
+    thinc-apple-ops>=1.0.0,<2.0.0
 # Language tokenizers with external dependencies
 ja =
     sudachipy>=0.5.2,!=0.6.1
     sudachidict_core>=20211220
 ko =
-    mecab-ko>=1.0.0
+    natto-py>=0.9.0
 th =
     pythainlp>=2.0
 
@@ -117,20 +132,13 @@ universal = false
 [sdist]
 formats = gztar
 
-[flake8]
-ignore = E203, E266, E501, E731, W503, E741, F541
-max-line-length = 80
-select = B,C,E,F,W,T4,B9
-exclude =
-    .env,
-    .git,
-    __pycache__,
-    _tokenizer_exceptions_list.py,
-
 [tool:pytest]
 markers =
     slow: mark a test as slow
     issue: reference specific issue
+filterwarnings =
+    error
+    ignore:Core Pydantic V1:UserWarning:pydantic
 
 [mypy]
 ignore_missing_imports = True

setup.py

@@ -37,6 +37,7 @@ MOD_NAMES = [
     "spacy.pipeline.dep_parser",
     "spacy.pipeline._edit_tree_internals.edit_trees",
     "spacy.pipeline.morphologizer",
+    "spacy.pipeline.multitask",
     "spacy.pipeline.ner",
     "spacy.pipeline.pipe",
     "spacy.pipeline.trainable_pipe",
@@ -47,7 +48,6 @@ MOD_NAMES = [
     "spacy.pipeline._parser_internals.arc_eager",
     "spacy.pipeline._parser_internals.ner",
     "spacy.pipeline._parser_internals.nonproj",
-    "spacy.pipeline._parser_internals.search",
     "spacy.pipeline._parser_internals._state",
     "spacy.pipeline._parser_internals.stateclass",
     "spacy.pipeline._parser_internals.transition_system",
@@ -61,13 +61,12 @@ MOD_NAMES = [
     "spacy.tokens.span_group",
     "spacy.tokens.graph",
     "spacy.tokens.morphanalysis",
-    "spacy.tokens.retokenizer",
+    "spacy.tokens._retokenize",
     "spacy.matcher.matcher",
     "spacy.matcher.phrasematcher",
     "spacy.matcher.dependencymatcher",
     "spacy.symbols",
     "spacy.vectors",
-    "spacy.tests.parser._search",
 ]
 COMPILE_OPTIONS = {
     "msvc": ["/Ox", "/EHsc"],
@@ -83,9 +82,9 @@ COMPILER_DIRECTIVES = {
 }
 # Files to copy into the package that are otherwise not included
 COPY_FILES = {
-    ROOT / "setup.cfg": PACKAGE_ROOT / "tests" / "package",
-    ROOT / "pyproject.toml": PACKAGE_ROOT / "tests" / "package",
-    ROOT / "requirements.txt": PACKAGE_ROOT / "tests" / "package",
+    ROOT / "setup.cfg": PACKAGE_ROOT / "tests" / "package" / "test.cfg",
+    ROOT / "pyproject.toml": PACKAGE_ROOT / "tests" / "package" / "test.toml",
+    ROOT / "requirements.txt": PACKAGE_ROOT / "tests" / "package" / "test.txt",
 }
 
 
@@ -159,10 +158,10 @@ GIT_VERSION = "%(git_version)s"
 
 
 def clean(path):
-    for path in path.glob("**/*"):
-        if path.is_file() and path.suffix in (".so", ".cpp", ".html"):
-            print(f"Deleting {path.name}")
-            path.unlink()
+    for child in path.glob("**/*"):
+        if child.is_file() and child.suffix in (".so", ".cpp", ".html"):
+            print(f"Deleting {child.name}")
+            child.unlink()
 
 
 def setup_package():
@@ -174,10 +173,10 @@ def setup_package():
         about = {}
         exec(f.read(), about)
 
-    for copy_file, target_dir in COPY_FILES.items():
+    for copy_file, target_file in COPY_FILES.items():
         if copy_file.exists():
-            shutil.copy(str(copy_file), str(target_dir))
-            print(f"Copied {copy_file} -> {target_dir}")
+            shutil.copyfile(str(copy_file), str(target_file))
+            print(f"Copied {copy_file} -> {target_file}")
 
     include_dirs = [
         numpy.get_include(),
@@ -214,7 +213,7 @@ def setup_package():
         version=about["__version__"],
         ext_modules=ext_modules,
         cmdclass={"build_ext": build_ext_subclass},
-        package_data={"": ["*.pyx", "*.pxd", "*.pxi"]},
+        package_data={"": ["*.pyx", "*.pxd", "*.pxi"], "spacy_cli": ["*.json"]},
     )
 
 

spacy/__init__.py

@@ -10,16 +10,39 @@ setup_default_warnings()  # noqa: E402
 # These are imported as part of the API
 from thinc.api import Config, prefer_gpu, require_cpu, require_gpu  # noqa: F401
 
-from . import pipeline  # noqa: F401
-from . import util
+from . import (
+    pipeline,  # noqa: F401
+    util,
+)
 from .about import __version__  # noqa: F401
 from .cli.info import info  # noqa: F401
 from .errors import Errors
 from .glossary import explain  # noqa: F401
 from .language import Language
+from .registrations import REGISTRY_POPULATED, populate_registry
+
+# Rebuild pydantic v2 schemas that use forward references to Language/Vocab
+from .schemas import (  # noqa: F401
+    ConfigSchema,
+    ConfigSchemaInit,
+    ConfigSchemaNlp,
+    ConfigSchemaPretrain,
+    ConfigSchemaTraining,
+)
+from .training import Example  # noqa: F401
 from .util import logger, registry  # noqa: F401
 from .vocab import Vocab
 
+_rebuild_ns = {"Language": Language, "Vocab": Vocab, "Example": Example}
+for _schema in (
+    ConfigSchemaTraining,
+    ConfigSchemaNlp,
+    ConfigSchemaPretrain,
+    ConfigSchemaInit,
+    ConfigSchema,
+):
+    _schema.model_rebuild(_types_namespace=_rebuild_ns)  # type: ignore[attr-defined]
+
 if sys.maxunicode == 65535:
     raise SystemError(Errors.E130)
 

spacy/about.py

@@ -1,9 +1,5 @@
 # fmt: off
 __title__ = "spacy"
-__version__ = "4.0.0.dev10"
+__version__ = "3.8.12"
 __download_url__ = "https://github.com/explosion/spacy-models/releases/download"
 __compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"
-__projects__ = "https://github.com/explosion/projects"
-__projects_branch__ = "v3"
-__lookups_tag__ = "v1.0.3"
-__lookups_url__ = f"https://raw.githubusercontent.com/explosion/spacy-lookups-data/{__lookups_tag__}/spacy_lookups_data/data/"

spacy/attrs.pxd

@@ -1,50 +1,99 @@
+# Reserve 64 values for flag features
 from . cimport symbols
 
 
 cdef enum attr_id_t:
-    NULL_ATTR = 0
-    IS_ALPHA = symbols.IS_ALPHA
-    IS_ASCII = symbols.IS_ASCII
-    IS_DIGIT = symbols.IS_DIGIT
-    IS_LOWER = symbols.IS_LOWER
-    IS_PUNCT = symbols.IS_PUNCT
-    IS_SPACE = symbols.IS_SPACE
-    IS_TITLE = symbols.IS_TITLE
-    IS_UPPER = symbols.IS_UPPER
-    LIKE_URL = symbols.LIKE_URL
-    LIKE_NUM = symbols.LIKE_NUM
-    LIKE_EMAIL = symbols.LIKE_EMAIL
-    IS_STOP = symbols.IS_STOP
-    IS_BRACKET = symbols.IS_BRACKET
-    IS_QUOTE = symbols.IS_QUOTE
-    IS_LEFT_PUNCT = symbols.IS_LEFT_PUNCT
-    IS_RIGHT_PUNCT = symbols.IS_RIGHT_PUNCT
-    IS_CURRENCY = symbols.IS_CURRENCY
+    NULL_ATTR
+    IS_ALPHA
+    IS_ASCII
+    IS_DIGIT
+    IS_LOWER
+    IS_PUNCT
+    IS_SPACE
+    IS_TITLE
+    IS_UPPER
+    LIKE_URL
+    LIKE_NUM
+    LIKE_EMAIL
+    IS_STOP
+    IS_OOV_DEPRECATED
+    IS_BRACKET
+    IS_QUOTE
+    IS_LEFT_PUNCT
+    IS_RIGHT_PUNCT
+    IS_CURRENCY
 
-    ID = symbols.ID
-    ORTH = symbols.ORTH
-    LOWER = symbols.LOWER
-    NORM = symbols.NORM
-    SHAPE = symbols.SHAPE
-    PREFIX = symbols.PREFIX
-    SUFFIX = symbols.SUFFIX
+    FLAG19 = 19
+    FLAG20
+    FLAG21
+    FLAG22
+    FLAG23
+    FLAG24
+    FLAG25
+    FLAG26
+    FLAG27
+    FLAG28
+    FLAG29
+    FLAG30
+    FLAG31
+    FLAG32
+    FLAG33
+    FLAG34
+    FLAG35
+    FLAG36
+    FLAG37
+    FLAG38
+    FLAG39
+    FLAG40
+    FLAG41
+    FLAG42
+    FLAG43
+    FLAG44
+    FLAG45
+    FLAG46
+    FLAG47
+    FLAG48
+    FLAG49
+    FLAG50
+    FLAG51
+    FLAG52
+    FLAG53
+    FLAG54
+    FLAG55
+    FLAG56
+    FLAG57
+    FLAG58
+    FLAG59
+    FLAG60
+    FLAG61
+    FLAG62
+    FLAG63
 
-    LENGTH = symbols.LENGTH
-    CLUSTER = symbols.CLUSTER
-    LEMMA = symbols.LEMMA
-    POS = symbols.POS
-    TAG = symbols.TAG
-    DEP = symbols.DEP
-    ENT_IOB = symbols.ENT_IOB
-    ENT_TYPE = symbols.ENT_TYPE
-    HEAD = symbols.HEAD
-    SENT_START = symbols.SENT_START
-    SPACY = symbols.SPACY
-    PROB = symbols.PROB
+    ID
+    ORTH
+    LOWER
+    NORM
+    SHAPE
+    PREFIX
+    SUFFIX
 
-    LANG = symbols.LANG
+    LENGTH
+    CLUSTER
+    LEMMA
+    POS
+    TAG
+    DEP
+    ENT_IOB
+    ENT_TYPE
+    HEAD
+    SENT_START
+    SPACY
+    PROB
+
+    LANG
     ENT_KB_ID = symbols.ENT_KB_ID
-    MORPH = symbols.MORPH
+    MORPH
     ENT_ID = symbols.ENT_ID
 
-    IDX = symbols.IDX
+    IDX
+    SENT_END

spacy/attrs.pyx

@@ -17,11 +17,57 @@ IDS = {
     "LIKE_NUM": LIKE_NUM,
     "LIKE_EMAIL": LIKE_EMAIL,
     "IS_STOP": IS_STOP,
+    "IS_OOV_DEPRECATED": IS_OOV_DEPRECATED,
     "IS_BRACKET": IS_BRACKET,
     "IS_QUOTE": IS_QUOTE,
     "IS_LEFT_PUNCT": IS_LEFT_PUNCT,
     "IS_RIGHT_PUNCT": IS_RIGHT_PUNCT,
     "IS_CURRENCY": IS_CURRENCY,
+    "FLAG19": FLAG19,
+    "FLAG20": FLAG20,
+    "FLAG21": FLAG21,
+    "FLAG22": FLAG22,
+    "FLAG23": FLAG23,
+    "FLAG24": FLAG24,
+    "FLAG25": FLAG25,
+    "FLAG26": FLAG26,
+    "FLAG27": FLAG27,
+    "FLAG28": FLAG28,
+    "FLAG29": FLAG29,
+    "FLAG30": FLAG30,
+    "FLAG31": FLAG31,
+    "FLAG32": FLAG32,
+    "FLAG33": FLAG33,
+    "FLAG34": FLAG34,
+    "FLAG35": FLAG35,
+    "FLAG36": FLAG36,
+    "FLAG37": FLAG37,
+    "FLAG38": FLAG38,
+    "FLAG39": FLAG39,
+    "FLAG40": FLAG40,
+    "FLAG41": FLAG41,
+    "FLAG42": FLAG42,
+    "FLAG43": FLAG43,
+    "FLAG44": FLAG44,
+    "FLAG45": FLAG45,
+    "FLAG46": FLAG46,
+    "FLAG47": FLAG47,
+    "FLAG48": FLAG48,
+    "FLAG49": FLAG49,
+    "FLAG50": FLAG50,
+    "FLAG51": FLAG51,
+    "FLAG52": FLAG52,
+    "FLAG53": FLAG53,
+    "FLAG54": FLAG54,
+    "FLAG55": FLAG55,
+    "FLAG56": FLAG56,
+    "FLAG57": FLAG57,
+    "FLAG58": FLAG58,
+    "FLAG59": FLAG59,
+    "FLAG60": FLAG60,
+    "FLAG61": FLAG61,
+    "FLAG62": FLAG62,
+    "FLAG63": FLAG63,
     "ID": ID,
     "ORTH": ORTH,
     "LOWER": LOWER,
@@ -47,11 +93,12 @@ IDS = {
 }
 
 
-NAMES = {v: k for k, v in IDS.items()}
+# ATTR IDs, in order of the symbol
+NAMES = [key for key, value in sorted(IDS.items(), key=lambda item: item[1])]
 locals().update(IDS)
 
 
-def intify_attrs(stringy_attrs, strings_map=None):
+def intify_attrs(stringy_attrs, strings_map=None, _do_deprecated=False):
     """
     Normalize a dictionary of attributes, converting them to ints.
 
@@ -63,6 +110,75 @@ def intify_attrs(stringy_attrs, strings_map=None):
         converted to ints.
     """
     inty_attrs = {}
+    if _do_deprecated:
+        if "F" in stringy_attrs:
+            stringy_attrs["ORTH"] = stringy_attrs.pop("F")
+        if "L" in stringy_attrs:
+            stringy_attrs["LEMMA"] = stringy_attrs.pop("L")
+        if "pos" in stringy_attrs:
+            stringy_attrs["TAG"] = stringy_attrs.pop("pos")
+        if "morph" in stringy_attrs:
+            morphs = stringy_attrs.pop("morph")  # no-cython-lint
+        if "number" in stringy_attrs:
+            stringy_attrs.pop("number")
+        if "tenspect" in stringy_attrs:
+            stringy_attrs.pop("tenspect")
+        morph_keys = [
+            "PunctType",
+            "PunctSide",
+            "Other",
+            "Degree",
+            "AdvType",
+            "Number",
+            "VerbForm",
+            "PronType",
+            "Aspect",
+            "Tense",
+            "PartType",
+            "Poss",
+            "Hyph",
+            "ConjType",
+            "NumType",
+            "Foreign",
+            "VerbType",
+            "NounType",
+            "Gender",
+            "Mood",
+            "Negative",
+            "Tense",
+            "Voice",
+            "Abbr",
+            "Derivation",
+            "Echo",
+            "Foreign",
+            "NameType",
+            "NounType",
+            "NumForm",
+            "NumValue",
+            "PartType",
+            "Polite",
+            "StyleVariant",
+            "PronType",
+            "AdjType",
+            "Person",
+            "Variant",
+            "AdpType",
+            "Reflex",
+            "Negative",
+            "Mood",
+            "Aspect",
+            "Case",
+            "Polarity",
+            "PrepCase",
+            "Animacy",  # U20
+        ]
+        for key in morph_keys:
+            if key in stringy_attrs:
+                stringy_attrs.pop(key)
+            elif key.lower() in stringy_attrs:
+                stringy_attrs.pop(key.lower())
+            elif key.upper() in stringy_attrs:
+                stringy_attrs.pop(key.upper())
     for name, value in stringy_attrs.items():
         int_key = intify_attr(name)
         if int_key is not None:

spacy/cli/__init__.py

@@ -1,41 +1,96 @@
+import sys
+import types
+from importlib import import_module
+from typing import Iterable
+
+from typer.main import get_command
 from wasabi import msg
 
-# Needed for testing
-from . import download as download_module  # noqa: F401
-from ._util import app, setup_cli  # noqa: F401
-from .apply import apply  # noqa: F401
-from .assemble import assemble_cli  # noqa: F401
-
-# These are the actual functions, NOT the wrapped CLI commands. The CLI commands
-# are registered automatically and won't have to be imported here.
-from .benchmark_speed import benchmark_speed_cli  # noqa: F401
-from .convert import convert  # noqa: F401
-from .debug_config import debug_config  # noqa: F401
-from .debug_data import debug_data  # noqa: F401
-from .debug_diff import debug_diff  # noqa: F401
-from .debug_model import debug_model  # noqa: F401
-from .distill import distill  # noqa: F401
-from .download import download  # noqa: F401
-from .evaluate import evaluate  # noqa: F401
-from .find_function import find_function  # noqa: F401
-from .find_threshold import find_threshold  # noqa: F401
-from .info import info  # noqa: F401
-from .init_config import fill_config, init_config  # noqa: F401
-from .init_pipeline import init_pipeline_cli  # noqa: F401
-from .package import package  # noqa: F401
-from .pretrain import pretrain  # noqa: F401
-from .profile import profile  # noqa: F401
-from .project.assets import project_assets  # type: ignore[attr-defined]  # noqa: F401
-from .project.clone import project_clone  # type: ignore[attr-defined]  # noqa: F401
-from .project.document import (  # type: ignore[attr-defined]  # noqa: F401
-    project_document,
+from ..util import registry
+from ._dispatch import (
+    GROUP_MODULES,
+    PUBLIC_ATTRS,
+    SUBCOMMAND_MODULES,
+    TOP_LEVEL_MODULES,
+    iter_builtin_modules,
 )
-from .project.dvc import project_update_dvc  # type: ignore[attr-defined]  # noqa: F401
-from .project.pull import project_pull  # type: ignore[attr-defined]  # noqa: F401
-from .project.push import project_push  # type: ignore[attr-defined]  # noqa: F401
-from .project.run import project_run  # type: ignore[attr-defined]  # noqa: F401
-from .train import train_cli  # type: ignore[attr-defined]  # noqa: F401
-from .validate import validate  # type: ignore[attr-defined]  # noqa: F401
+from ._util import COMMAND, add_project_cli, app
+
+HELP_OPTIONS = {"--help", "-h"}
+ROOT_OPTIONS = HELP_OPTIONS | {"--install-completion", "--show-completion"}
+
+__all__ = [
+    "app",
+    "load_all_commands",
+    "load_for_argv",
+    "setup_cli",
+    *sorted(PUBLIC_ATTRS),
+]
+
+
+def _import_modules(module_names: Iterable[str]) -> None:
+    for module_name in module_names:
+        import_module(module_name)
+
+
+def load_all_commands() -> None:
+    _import_modules(iter_builtin_modules())
+    add_project_cli()
+
+
+def load_for_argv(argv: Iterable[str]) -> None:
+    args = list(argv)
+    if not args or args[0] in ROOT_OPTIONS or args[0].startswith("-"):
+        load_all_commands()
+        return
+    command = args[0]
+    if command == "project":
+        add_project_cli()
+        return
+    if command in GROUP_MODULES:
+        subcommand = args[1] if len(args) > 1 and not args[1].startswith("-") else None
+        if subcommand is not None and (command, subcommand) in SUBCOMMAND_MODULES:
+            _import_modules(SUBCOMMAND_MODULES[(command, subcommand)])
+            return
+        _import_modules(GROUP_MODULES[command])
+        return
+    if command in TOP_LEVEL_MODULES:
+        _import_modules(TOP_LEVEL_MODULES[command])
+
+
+def setup_cli() -> None:
+    # Make sure entry-point CLI integrations are imported before command dispatch.
+    registry.cli.get_all()
+    load_for_argv(sys.argv[1:])
+    command = get_command(app)
+    command(prog_name=COMMAND)
+
+
+def __getattr__(name: str):
+    if name not in PUBLIC_ATTRS:
+        raise AttributeError(f"module 'spacy.cli' has no attribute {name!r}")
+    module_name, attr_name = PUBLIC_ATTRS[name]
+    module = import_module(module_name)
+    value = module if attr_name is None else getattr(module, attr_name)
+    globals()[name] = value
+    return value
+
+
+def __dir__():
+    return sorted(set(globals()) | set(PUBLIC_ATTRS))
+
+
+class _CLIModule(types.ModuleType):
+    def __setattr__(self, name, value):
+        if isinstance(value, types.ModuleType) and name in PUBLIC_ATTRS:
+            _, attr_name = PUBLIC_ATTRS[name]
+            if attr_name is not None:
+                super().__setattr__(name, getattr(value, attr_name))
+                return
+        super().__setattr__(name, value)
+
+
+sys.modules[__name__].__class__ = _CLIModule
 
 
 @app.command("link", no_args_is_help=True, deprecated=True, hidden=True)

spacy/cli/_dispatch.py

@@ -0,0 +1,104 @@
+from typing import Dict, Iterable, Optional, Tuple
+
+CommandPath = Tuple[str, ...]
+
+
+TOP_LEVEL_MODULES: Dict[str, Tuple[str, ...]] = {
+    "apply": ("spacy.cli.apply",),
+    "assemble": ("spacy.cli.assemble",),
+    "convert": ("spacy.cli.convert",),
+    "debug-data": ("spacy.cli.debug_data",),
+    "download": ("spacy.cli.download",),
+    "evaluate": ("spacy.cli.evaluate",),
+    "find-function": ("spacy.cli.find_function",),
+    "find-threshold": ("spacy.cli.find_threshold",),
+    "info": ("spacy.cli.info",),
+    "package": ("spacy.cli.package",),
+    "pretrain": ("spacy.cli.pretrain",),
+    "profile": ("spacy.cli.profile",),
+    "train": ("spacy.cli.train",),
+    "validate": ("spacy.cli.validate",),
+}
+
+
+GROUP_MODULES: Dict[str, Tuple[str, ...]] = {
+    "benchmark": (
+        "spacy.cli.benchmark_speed",
+        "spacy.cli.evaluate",
+    ),
+    "debug": (
+        "spacy.cli.debug_config",
+        "spacy.cli.debug_data",
+        "spacy.cli.debug_diff",
+        "spacy.cli.debug_model",
+        "spacy.cli.profile",
+    ),
+    "init": (
+        "spacy.cli.init_config",
+        "spacy.cli.init_pipeline",
+    ),
+}
+
+
+SUBCOMMAND_MODULES: Dict[CommandPath, Tuple[str, ...]] = {
+    ("benchmark", "accuracy"): ("spacy.cli.evaluate",),
+    ("benchmark", "speed"): ("spacy.cli.benchmark_speed",),
+    ("debug", "config"): ("spacy.cli.debug_config",),
+    ("debug", "data"): ("spacy.cli.debug_data",),
+    ("debug", "diff-config"): ("spacy.cli.debug_diff",),
+    ("debug", "model"): ("spacy.cli.debug_model",),
+    ("debug", "profile"): ("spacy.cli.profile",),
+    ("init", "config"): ("spacy.cli.init_config",),
+    ("init", "fill-config"): ("spacy.cli.init_config",),
+    ("init", "labels"): ("spacy.cli.init_pipeline",),
+    ("init", "nlp"): ("spacy.cli.init_pipeline",),
+    ("init", "vectors"): ("spacy.cli.init_pipeline",),
+}
+
+
+PUBLIC_ATTRS: Dict[str, Tuple[str, Optional[str]]] = {
+    "app": ("spacy.cli._util", "app"),
+    "apply": ("spacy.cli.apply", "apply"),
+    "assemble_cli": ("spacy.cli.assemble", "assemble_cli"),
+    "benchmark_speed_cli": ("spacy.cli.benchmark_speed", "benchmark_speed_cli"),
+    "convert": ("spacy.cli.convert", "convert"),
+    "debug_config": ("spacy.cli.debug_config", "debug_config"),
+    "debug_data": ("spacy.cli.debug_data", "debug_data"),
+    "debug_diff": ("spacy.cli.debug_diff", "debug_diff"),
+    "debug_model": ("spacy.cli.debug_model", "debug_model"),
+    "download": ("spacy.cli.download", "download"),
+    "download_module": ("spacy.cli.download", None),
+    "evaluate": ("spacy.cli.evaluate", "evaluate"),
+    "fill_config": ("spacy.cli.init_config", "fill_config"),
+    "find_function": ("spacy.cli.find_function", "find_function"),
+    "find_threshold": ("spacy.cli.find_threshold", "find_threshold"),
+    "info": ("spacy.cli.info", "info"),
+    "init_config": ("spacy.cli.init_config", "init_config"),
+    "init_pipeline_cli": ("spacy.cli.init_pipeline", "init_pipeline_cli"),
+    "package": ("spacy.cli.package", "package"),
+    "pretrain": ("spacy.cli.pretrain", "pretrain"),
+    "profile": ("spacy.cli.profile", "profile"),
+    "project_assets": ("spacy.cli.project.assets", "project_assets"),
+    "project_clone": ("spacy.cli.project.clone", "project_clone"),
+    "project_document": ("spacy.cli.project.document", "project_document"),
+    "project_pull": ("spacy.cli.project.pull", "project_pull"),
+    "project_push": ("spacy.cli.project.push", "project_push"),
+    "project_run": ("spacy.cli.project.run", "project_run"),
+    "project_update_dvc": ("spacy.cli.project.dvc", "project_update_dvc"),
+    "train_cli": ("spacy.cli.train", "train_cli"),
+    "validate": ("spacy.cli.validate", "validate"),
+}
+
+
+def iter_builtin_modules() -> Iterable[str]:
+    seen = set()
+    for modules in TOP_LEVEL_MODULES.values():
+        for module in modules:
+            if module not in seen:
+                seen.add(module)
+                yield module
+    for modules in GROUP_MODULES.values():
+        for module in modules:
+            if module not in seen:
+                seen.add(module)
+                yield module

spacy/cli/_util.py

@@ -1,17 +1,12 @@
-import hashlib
 import os
-import shutil
 import sys
 from configparser import InterpolationError
 from contextlib import contextmanager
 from pathlib import Path
 from typing import (
-    TYPE_CHECKING,
     Any,
     Dict,
-    Iterable,
     List,
-    Literal,
     Optional,
     Tuple,
     Union,
@@ -21,24 +16,16 @@ from typing import (
 import srsly
 import typer
 from click import NoSuchOption
-from click.parser import split_arg_string
-from thinc.api import Config, ConfigValidationError, require_gpu
+from click.shell_completion import split_arg_string
+from thinc.api import ConfigValidationError, require_gpu
 from thinc.util import gpu_is_available
-from typer.main import get_command
 from wasabi import Printer, msg
-from weasel import app as project_cli
 
-from .. import about
-from ..errors import RENAMED_LANGUAGE_CODES
-from ..schemas import validate
+from ..compat import Literal
 from ..util import (
     ENV_VARS,
-    SimpleFrozenDict,
     import_file,
-    is_compatible_version,
     logger,
-    make_tempdir,
-    registry,
     run_command,
 )
 
@@ -69,23 +56,25 @@ INIT_HELP = """Commands for initializing configs and pipeline packages."""
 Arg = typer.Argument
 Opt = typer.Option
 
-app = typer.Typer(name=NAME, help=HELP)
+app = typer.Typer(name=NAME, help=HELP, rich_markup_mode=None)
 benchmark_cli = typer.Typer(name="benchmark", help=BENCHMARK_HELP, no_args_is_help=True)
 debug_cli = typer.Typer(name="debug", help=DEBUG_HELP, no_args_is_help=True)
 init_cli = typer.Typer(name="init", help=INIT_HELP, no_args_is_help=True)
+_PROJECT_CLI_ADDED = False
 
-app.add_typer(project_cli, name="project", help=PROJECT_HELP, no_args_is_help=True)
 app.add_typer(debug_cli)
 app.add_typer(benchmark_cli)
 app.add_typer(init_cli)
 
 
-def setup_cli() -> None:
-    # Make sure the entry-point for CLI runs, so that they get imported.
-    registry.cli.get_all()
-    # Ensure that the help messages always display the correct prompt
-    command = get_command(app)
-    command(prog_name=COMMAND)
+def add_project_cli() -> None:
+    global _PROJECT_CLI_ADDED
+    if _PROJECT_CLI_ADDED:
+        return
+    from weasel import app as project_cli
+
+    app.add_typer(project_cli, name="project", help=PROJECT_HELP, no_args_is_help=True)
+    _PROJECT_CLI_ADDED = True
 
 
 def parse_config_overrides(
@@ -149,16 +138,6 @@ def _parse_override(value: Any) -> Any:
         return str(value)
 
 
-def _handle_renamed_language_codes(lang: Optional[str]) -> None:
-    # Throw error for renamed language codes in v4
-    if lang in RENAMED_LANGUAGE_CODES:
-        msg.fail(
-            title="Renamed language code",
-            text=f"Language code '{lang}' was replaced with '{RENAMED_LANGUAGE_CODES[lang]}' in spaCy v4. Update the language code from '{lang}' to '{RENAMED_LANGUAGE_CODES[lang]}'.",
-            exits=1,
-        )
-
-
 @contextmanager
 def show_validation_error(
     file_path: Optional[Union[str, Path]] = None,
@@ -203,13 +182,6 @@ def show_validation_error(
         msg.fail("Config validation error", e, exits=1)
 
 
-def import_code_paths(code_paths: str) -> None:
-    """Helper to import comma-separated list of code paths."""
-    code_paths = [Path(p.strip()) for p in string_to_list(code_paths)]
-    for code_path in code_paths:
-        import_code(code_path)
-
-
 def import_code(code_path: Optional[Union[Path, str]]) -> None:
     """Helper to import Python file provided in training commands / commands
     using the config. This makes custom registered functions available.
@@ -233,8 +205,8 @@ def get_git_version(
     """
     try:
         ret = run_command("git --version", capture=True)
-    except:
-        raise RuntimeError(error)
+    except Exception as err:
+        raise RuntimeError(error) from err
     stdout = ret.stdout.strip()
     if not stdout or not stdout.startswith("git version"):
         return 0, 0
@@ -243,13 +215,11 @@ def get_git_version(
 
 
 @overload
-def string_to_list(value: str, intify: Literal[False] = ...) -> List[str]:
-    ...
+def string_to_list(value: str, intify: Literal[False] = ...) -> List[str]: ...
 
 
 @overload
-def string_to_list(value: str, intify: Literal[True]) -> List[int]:
-    ...
+def string_to_list(value: str, intify: Literal[True]) -> List[int]: ...
 
 
 def string_to_list(value: str, intify: bool = False) -> Union[List[str], List[int]]:

spacy/cli/apply.py

@@ -22,7 +22,7 @@ to be grabbed ("text" by default)."""
 
 out_help = "Path to save the resulting .spacy file"
 code_help = (
-    "Path to Python file with additional " "code (registered functions) to be imported"
+    "Path to Python file with additional code (registered functions) to be imported"
 )
 gold_help = "Use gold preprocessing provided in the .spacy files"
 force_msg = (
@@ -72,11 +72,15 @@ def apply_cli(
     data_path: Path = Arg(..., help=path_help, exists=True),
     output_file: Path = Arg(..., help=out_help, dir_okay=False),
     code_path: Optional[Path] = Opt(None, "--code", "-c", help=code_help),
-    text_key: str = Opt("text", "--text-key", "-tk", help="Key containing text string for JSONL"),
-    force_overwrite: bool = Opt(False, "--force", "-F", help="Force overwriting the output file"),
+    text_key: str = Opt(
+        "text", "--text-key", "-tk", help="Key containing text string for JSONL"
+    ),
+    force_overwrite: bool = Opt(
+        False, "--force", "-F", help="Force overwriting the output file"
+    ),
     use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU."),
     batch_size: int = Opt(1, "--batch-size", "-b", help="Batch size."),
-    n_process: int = Opt(1, "--n-process", "-n", help="number of processors to use.")
+    n_process: int = Opt(1, "--n-process", "-n", help="number of processors to use."),
 ):
     """
     Apply a trained pipeline to documents to get predictions.
@@ -114,8 +118,7 @@ def apply(
     if len(paths) == 0:
         docbin.to_disk(output_file)
         msg.warn(
-            "Did not find data to process,"
-            f" {data_path} seems to be an empty directory."
+            f"Did not find data to process, {data_path} seems to be an empty directory."
         )
         return
     nlp = load_model(model)

spacy/cli/assemble.py

@@ -11,7 +11,7 @@ from ._util import (
     Arg,
     Opt,
     app,
-    import_code_paths,
+    import_code,
     parse_config_overrides,
     show_validation_error,
 )
@@ -24,10 +24,25 @@ from ._util import (
 def assemble_cli(
     # fmt: off
     ctx: typer.Context,  # This is only used to read additional arguments
-    config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
-    output_path: Path = Arg(..., help="Output directory to store assembled pipeline in"),
-    code_path: str = Opt("", "--code", "-c", help="Comma-separated paths to Python files with additional code (registered functions) to be imported"),
-    verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
+    config_path: Path = Arg(
+        ..., help="Path to config file", exists=True, allow_dash=True
+    ),
+    output_path: Path = Arg(
+        ..., help="Output directory to store assembled pipeline in"
+    ),
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
+    verbose: bool = Opt(
+        False,
+        "--verbose",
+        "-V",
+        "-VV",
+        help="Display more information for debugging purposes",
+    ),
     # fmt: on
 ):
     """
@@ -46,7 +61,7 @@ def assemble_cli(
     if not config_path or (str(config_path) != "-" and not config_path.exists()):
         msg.fail("Config file not found", config_path, exits=1)
     overrides = parse_config_overrides(ctx.args)
-    import_code_paths(code_path)
+    import_code(code_path)
     with show_validation_error(config_path):
         config = util.load_config(config_path, overrides=overrides, interpolate=False)
     msg.divider("Initializing pipeline")

spacy/cli/benchmark_speed.py

@@ -24,13 +24,29 @@ def benchmark_speed_cli(
     # fmt: off
     ctx: typer.Context,
     model: str = Arg(..., help="Model name or path"),
-    data_path: Path = Arg(..., help="Location of binary evaluation data in .spacy format", exists=True),
-    batch_size: Optional[int] = Opt(None, "--batch-size", "-b", min=1, help="Override the pipeline batch size"),
+    data_path: Path = Arg(
+        ..., help="Location of binary evaluation data in .spacy format", exists=True
+    ),
+    batch_size: Optional[int] = Opt(
+        None, "--batch-size", "-b", min=1, help="Override the pipeline batch size"
+    ),
     no_shuffle: bool = Opt(False, "--no-shuffle", help="Do not shuffle benchmark data"),
     use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
-    n_batches: int = Opt(50, "--batches", help="Minimum number of batches to benchmark", min=30,),
-    warmup_epochs: int = Opt(3, "--warmup", "-w", min=0, help="Number of iterations over the data for warmup"),
-    code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
+    n_batches: int = Opt(
+        50,
+        "--batches",
+        help="Minimum number of batches to benchmark",
+        min=30,
+    ),
+    warmup_epochs: int = Opt(
+        3, "--warmup", "-w", min=0, help="Number of iterations over the data for warmup"
+    ),
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
     # fmt: on
 ):
     """
@@ -151,7 +167,7 @@ def print_mean_with_ci(sample: numpy.ndarray):
     low = bootstrap_means[int(len(bootstrap_means) * 0.025)]
     high = bootstrap_means[int(len(bootstrap_means) * 0.975)]
 
-    print(f"Mean: {mean:.1f} words/s (95% CI: {low-mean:.1f} +{high-mean:.1f})")
+    print(f"Mean: {mean:.1f} words/s (95% CI: {low - mean:.1f} +{high - mean:.1f})")
 
 
 def print_outliers(sample: numpy.ndarray):

spacy/cli/convert.py

@@ -16,7 +16,7 @@ from ..training.converters import (
     iob_to_docs,
     json_to_docs,
 )
-from ._util import Arg, Opt, _handle_renamed_language_codes, app, walk_directory
+from ._util import Arg, Opt, app, walk_directory
 
 # Converters are matched by file extension except for ner/iob, which are
 # matched by file extension and content. To add a converter, add a new
@@ -48,17 +48,47 @@ class FileTypes(str, Enum):
 def convert_cli(
     # fmt: off
     input_path: str = Arg(..., help="Input file or directory", exists=True),
-    output_dir: Path = Arg("-", help="Output directory. '-' for stdout.", allow_dash=True, exists=True),
-    file_type: FileTypes = Opt("spacy", "--file-type", "-t", help="Type of data to produce"),
-    n_sents: int = Opt(1, "--n-sents", "-n", help="Number of sentences per doc (0 to disable)"),
-    seg_sents: bool = Opt(False, "--seg-sents", "-s", help="Segment sentences (for -c ner)"),
-    model: Optional[str] = Opt(None, "--model", "--base", "-b", help="Trained spaCy pipeline for sentence segmentation to use as base (for --seg-sents)"),
-    morphology: bool = Opt(False, "--morphology", "-m", help="Enable appending morphology to tags"),
-    merge_subtokens: bool = Opt(False, "--merge-subtokens", "-T", help="Merge CoNLL-U subtokens"),
-    converter: str = Opt(AUTO, "--converter", "-c", help=f"Converter: {tuple(CONVERTERS.keys())}"),
-    ner_map: Optional[Path] = Opt(None, "--ner-map", "-nm", help="NER tag mapping (as JSON-encoded dict of entity types)", exists=True),
-    lang: Optional[str] = Opt(None, "--lang", "-l", help="Language (if tokenizer required)"),
-    concatenate: bool = Opt(None, "--concatenate", "-C", help="Concatenate output to a single file"),
+    output_dir: Path = Arg(
+        "-", help="Output directory. '-' for stdout.", allow_dash=True, exists=True
+    ),
+    file_type: FileTypes = Opt(
+        "spacy", "--file-type", "-t", help="Type of data to produce"
+    ),
+    n_sents: int = Opt(
+        1, "--n-sents", "-n", help="Number of sentences per doc (0 to disable)"
+    ),
+    seg_sents: bool = Opt(
+        False, "--seg-sents", "-s", help="Segment sentences (for -c ner)"
+    ),
+    model: Optional[str] = Opt(
+        None,
+        "--model",
+        "--base",
+        "-b",
+        help="Trained spaCy pipeline for sentence segmentation to use as base (for --seg-sents)",
+    ),
+    morphology: bool = Opt(
+        False, "--morphology", "-m", help="Enable appending morphology to tags"
+    ),
+    merge_subtokens: bool = Opt(
+        False, "--merge-subtokens", "-T", help="Merge CoNLL-U subtokens"
+    ),
+    converter: str = Opt(
+        AUTO, "--converter", "-c", help=f"Converter: {tuple(CONVERTERS.keys())}"
+    ),
+    ner_map: Optional[Path] = Opt(
+        None,
+        "--ner-map",
+        "-nm",
+        help="NER tag mapping (as JSON-encoded dict of entity types)",
+        exists=True,
+    ),
+    lang: Optional[str] = Opt(
+        None, "--lang", "-l", help="Language (if tokenizer required)"
+    ),
+    concatenate: bool = Opt(
+        None, "--concatenate", "-C", help="Concatenate output to a single file"
+    ),
     # fmt: on
 ):
     """
@@ -116,10 +146,6 @@ def convert(
     input_path = Path(input_path)
     if not msg:
         msg = Printer(no_print=silent)
-
-    # Throw error for renamed language codes in v4
-    _handle_renamed_language_codes(lang)
-
     ner_map = srsly.read_json(ner_map) if ner_map is not None else None
     doc_files = []
     for input_loc in walk_directory(input_path, converter):

spacy/cli/debug_config.py

@@ -13,7 +13,7 @@ from ._util import (
     Arg,
     Opt,
     debug_cli,
-    import_code_paths,
+    import_code,
     parse_config_overrides,
     show_validation_error,
 )
@@ -26,10 +26,28 @@ from ._util import (
 def debug_config_cli(
     # fmt: off
     ctx: typer.Context,  # This is only used to read additional arguments
-    config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
-    code_path: str = Opt("", "--code", "-c", help="Comma-separated paths to Python files with additional code (registered functions) to be imported"),
-    show_funcs: bool = Opt(False, "--show-functions", "-F", help="Show an overview of all registered functions used in the config and where they come from (modules, files etc.)"),
-    show_vars: bool = Opt(False, "--show-variables", "-V", help="Show an overview of all variables referenced in the config and their values. This will also reflect variables overwritten on the CLI.")
+    config_path: Path = Arg(
+        ..., help="Path to config file", exists=True, allow_dash=True
+    ),
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code-path",
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
+    show_funcs: bool = Opt(
+        False,
+        "--show-functions",
+        "-F",
+        help="Show an overview of all registered functions used in the config and where they come from (modules, files etc.)",
+    ),
+    show_vars: bool = Opt(
+        False,
+        "--show-variables",
+        "-V",
+        help="Show an overview of all variables referenced in the config and their values. This will also reflect variables overwritten on the CLI.",
+    ),
     # fmt: on
 ):
     """Debug a config file and show validation errors. The command will
@@ -44,7 +62,7 @@ def debug_config_cli(
     DOCS: https://spacy.io/api/cli#debug-config
     """
     overrides = parse_config_overrides(ctx.args)
-    import_code_paths(code_path)
+    import_code(code_path)
     debug_config(
         config_path, overrides=overrides, show_funcs=show_funcs, show_vars=show_vars
     )
@@ -64,10 +82,10 @@ def debug_config(
         config = nlp.config.interpolate()
     msg.divider("Config validation for [initialize]")
     with show_validation_error(config_path):
-        T = registry.resolve(config["initialize"], schema=ConfigSchemaInit)
+        T = registry.resolve(config["initialize"], schema=ConfigSchemaInit)  # type: ignore[arg-type]
     msg.divider("Config validation for [training]")
     with show_validation_error(config_path):
-        T = registry.resolve(config["training"], schema=ConfigSchemaTraining)
+        T = registry.resolve(config["training"], schema=ConfigSchemaTraining)  # type: ignore[arg-type]
         dot_names = [T["train_corpus"], T["dev_corpus"]]
         util.resolve_dot_names(config, dot_names)
     msg.good("Config is valid")

spacy/cli/debug_data.py

@@ -7,7 +7,6 @@ from typing import (
     Dict,
     Iterable,
     List,
-    Literal,
     Optional,
     Sequence,
     Set,
@@ -23,6 +22,7 @@ import typer
 from wasabi import MESSAGES, Printer, msg
 
 from .. import util
+from ..compat import Literal
 from ..language import Language
 from ..morphology import Morphology
 from ..pipeline import Morphologizer, SpanCategorizer, TrainablePipe
@@ -40,7 +40,7 @@ from ._util import (
     _format_number,
     app,
     debug_cli,
-    import_code_paths,
+    import_code,
     parse_config_overrides,
     show_validation_error,
 )
@@ -71,11 +71,28 @@ SPAN_LENGTH_THRESHOLD_PERCENTAGE = 90
 def debug_data_cli(
     # fmt: off
     ctx: typer.Context,  # This is only used to read additional arguments
-    config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
-    code_path: str = Opt("", "--code", "-c", help="Comma-separated paths to Python files with additional code (registered functions) to be imported"),
-    ignore_warnings: bool = Opt(False, "--ignore-warnings", "-IW", help="Ignore warnings, only show stats and errors"),
-    verbose: bool = Opt(False, "--verbose", "-V", help="Print additional information and explanations"),
-    no_format: bool = Opt(False, "--no-format", "-NF", help="Don't pretty-print the results"),
+    config_path: Path = Arg(
+        ..., help="Path to config file", exists=True, allow_dash=True
+    ),
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code-path",
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
+    ignore_warnings: bool = Opt(
+        False,
+        "--ignore-warnings",
+        "-IW",
+        help="Ignore warnings, only show stats and errors",
+    ),
+    verbose: bool = Opt(
+        False, "--verbose", "-V", help="Print additional information and explanations"
+    ),
+    no_format: bool = Opt(
+        False, "--no-format", "-NF", help="Don't pretty-print the results"
+    ),
     # fmt: on
 ):
     """
@@ -92,7 +109,7 @@ def debug_data_cli(
             "--help for an overview of the other available debugging commands."
         )
     overrides = parse_config_overrides(ctx.args)
-    import_code_paths(code_path)
+    import_code(code_path)
     debug_data(
         config_path,
         config_overrides=overrides,
@@ -120,7 +137,7 @@ def debug_data(
         cfg = util.load_config(config_path, overrides=config_overrides)
         nlp = util.load_model_from_config(cfg)
         config = nlp.config.interpolate()
-        T = registry.resolve(config["training"], schema=ConfigSchemaTraining)
+        T = registry.resolve(config["training"], schema=ConfigSchemaTraining)  # type: ignore[arg-type]
     # Use original config here, not resolved version
     sourced_components = get_sourced_components(cfg)
     frozen_components = T["frozen_components"]
@@ -708,7 +725,7 @@ def debug_data(
         if len(dev_not_train) != 0:
             pct = len(dev_not_train) / len(trees_dev)
             msg.info(
-                f"{len(dev_not_train)} lemmatizer trees ({pct*100:.1f}% of dev trees)"
+                f"{len(dev_not_train)} lemmatizer trees ({pct * 100:.1f}% of dev trees)"
                 " were found exclusively in the dev data."
             )
         else:
@@ -968,16 +985,14 @@ def _compile_gold(
 
 
 @overload
-def _format_labels(labels: Iterable[str], counts: Literal[False] = False) -> str:
-    ...
+def _format_labels(labels: Iterable[str], counts: Literal[False] = False) -> str: ...
 
 
 @overload
 def _format_labels(
     labels: Iterable[Tuple[str, int]],
     counts: Literal[True],
-) -> str:
-    ...
+) -> str: ...
 
 
 def _format_labels(
@@ -1073,7 +1088,8 @@ def _get_distribution(docs, normalize: bool = True) -> Counter:
     word_counts: Counter = Counter()
     for doc in docs:
         for token in doc:
-            t = token.text.lower()
+            # Normalize the text
+            t = token.text.lower().replace("``", '"').replace("''", '"')
             word_counts[t] += 1
     if normalize:
         total = sum(word_counts.values(), 0.0)

spacy/cli/debug_diff.py

@@ -2,11 +2,10 @@ from pathlib import Path
 from typing import Optional
 
 import typer
-from thinc.api import Config
 from wasabi import MarkdownRenderer, Printer, diff_strings
 
 from ..util import load_config
-from ._util import Arg, Opt, debug_cli, parse_config_overrides, show_validation_error
+from ._util import Arg, Opt, debug_cli, show_validation_error
 from .init_config import Optimizations, init_config
 
 
@@ -17,12 +16,36 @@ from .init_config import Optimizations, init_config
 def debug_diff_cli(
     # fmt: off
     ctx: typer.Context,
-    config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
-    compare_to: Optional[Path] = Opt(None, help="Path to a config file to diff against, or `None` to compare against default settings", exists=True, allow_dash=True),
-    optimize: Optimizations = Opt(Optimizations.efficiency.value, "--optimize", "-o", help="Whether the user config was optimized for efficiency or accuracy. Only relevant when comparing against the default config."),
-    gpu: bool = Opt(False, "--gpu", "-G", help="Whether the original config can run on a GPU. Only relevant when comparing against the default config."),
-    pretraining: bool = Opt(False, "--pretraining", "--pt", help="Whether to compare on a config with pretraining involved. Only relevant when comparing against the default config."),
-    markdown: bool = Opt(False, "--markdown", "-md", help="Generate Markdown for GitHub issues")
+    config_path: Path = Arg(
+        ..., help="Path to config file", exists=True, allow_dash=True
+    ),
+    compare_to: Optional[Path] = Opt(
+        None,
+        help="Path to a config file to diff against, or `None` to compare against default settings",
+        exists=True,
+        allow_dash=True,
+    ),
+    optimize: Optimizations = Opt(
+        Optimizations.efficiency.value,
+        "--optimize",
+        "-o",
+        help="Whether the user config was optimized for efficiency or accuracy. Only relevant when comparing against the default config.",
+    ),
+    gpu: bool = Opt(
+        False,
+        "--gpu",
+        "-G",
+        help="Whether the original config can run on a GPU. Only relevant when comparing against the default config.",
+    ),
+    pretraining: bool = Opt(
+        False,
+        "--pretraining",
+        "--pt",
+        help="Whether to compare on a config with pretraining involved. Only relevant when comparing against the default config.",
+    ),
+    markdown: bool = Opt(
+        False, "--markdown", "-md", help="Generate Markdown for GitHub issues"
+    ),
     # fmt: on
 ):
     """Show a diff of a config file with respect to spaCy's defaults or another config file. If

spacy/cli/debug_model.py

@@ -36,18 +36,26 @@ from ._util import (
 def debug_model_cli(
     # fmt: off
     ctx: typer.Context,  # This is only used to read additional arguments
-    config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
-    component: str = Arg(..., help="Name of the pipeline component of which the model should be analysed"),
-    layers: str = Opt("", "--layers", "-l", help="Comma-separated names of layer IDs to print"),
+    config_path: Path = Arg(
+        ..., help="Path to config file", exists=True, allow_dash=True
+    ),
+    component: str = Arg(
+        ..., help="Name of the pipeline component of which the model should be analysed"
+    ),
+    layers: str = Opt(
+        "", "--layers", "-l", help="Comma-separated names of layer IDs to print"
+    ),
     dimensions: bool = Opt(False, "--dimensions", "-DIM", help="Show dimensions"),
     parameters: bool = Opt(False, "--parameters", "-PAR", help="Show parameters"),
     gradients: bool = Opt(False, "--gradients", "-GRAD", help="Show gradients"),
     attributes: bool = Opt(False, "--attributes", "-ATTR", help="Show attributes"),
     P0: bool = Opt(False, "--print-step0", "-P0", help="Print model before training"),
-    P1: bool = Opt(False, "--print-step1", "-P1", help="Print model after initialization"),
+    P1: bool = Opt(
+        False, "--print-step1", "-P1", help="Print model after initialization"
+    ),
     P2: bool = Opt(False, "--print-step2", "-P2", help="Print model after training"),
     P3: bool = Opt(False, "--print-step3", "-P3", help="Print final predictions"),
-    use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU")
+    use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
     # fmt: on
 ):
     """
@@ -81,7 +89,7 @@ def debug_model_cli(
     with show_validation_error(config_path):
         nlp = util.load_model_from_config(raw_config)
         config = nlp.config.interpolate()
-        T = registry.resolve(config["training"], schema=ConfigSchemaTraining)
+        T = registry.resolve(config["training"], schema=ConfigSchemaTraining)  # type: ignore[arg-type]
     seed = T["seed"]
     if seed is not None:
         msg.info(f"Fixing random seed: {seed}")
@@ -170,7 +178,7 @@ def debug_model(
         msg.divider(f"STEP 3 - prediction")
         msg.info(str(prediction))
 
-    msg.good(f"Succesfully ended analysis - model looks good.")
+    msg.good(f"Successfully ended analysis - model looks good.")
 
 
 def _sentences():

spacy/cli/distill.py

@@ -1,98 +0,0 @@
-import logging
-import sys
-from pathlib import Path
-from typing import Any, Dict, Optional, Union
-
-import typer
-from wasabi import msg
-
-from .. import util
-from ..pipeline.trainable_pipe import TrainablePipe
-from ..schemas import ConfigSchemaDistill
-from ..training.initialize import init_nlp_student
-from ..training.loop import distill as distill_nlp
-from ._util import (
-    Arg,
-    Opt,
-    app,
-    import_code_paths,
-    parse_config_overrides,
-    setup_gpu,
-    show_validation_error,
-)
-
-
-@app.command(
-    "distill",
-    context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
-)
-def distill_cli(
-    # fmt: off
-    ctx: typer.Context,  # This is only used to read additional arguments
-    teacher_model: str = Arg(..., help="Teacher model name or path"),
-    student_config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
-    output_path: Optional[Path] = Opt(None, "--output", "--output-path", "-o", help="Output directory to store trained pipeline in"),
-    code_path: str = Opt("", "--code", "-c", help="Comma-separated paths to Python files with additional code (registered functions) to be imported"),
-    verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
-    use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU")
-    # fmt: on
-):
-    """
-    Distill a spaCy pipeline from a teacher model.
-
-    DOCS: https://spacy.io/api/cli#distill
-    """
-    util.logger.setLevel(logging.DEBUG if verbose else logging.INFO)
-    overrides = parse_config_overrides(ctx.args)
-    import_code_paths(code_path)
-    distill(
-        teacher_model,
-        student_config_path,
-        output_path,
-        use_gpu=use_gpu,
-        overrides=overrides,
-    )
-
-
-def distill(
-    teacher_model: Union[str, Path],
-    student_config_path: Union[str, Path],
-    output_path: Optional[Union[str, Path]] = None,
-    *,
-    use_gpu: int = -1,
-    overrides: Dict[str, Any] = util.SimpleFrozenDict(),
-):
-    student_config_path = util.ensure_path(student_config_path)
-    output_path = util.ensure_path(output_path)
-    # Make sure all files and paths exist if they are needed
-    if not student_config_path or (
-        str(student_config_path) != "-" and not student_config_path.exists()
-    ):
-        msg.fail("Student config file not found", student_config_path, exits=1)
-    if not output_path:
-        msg.info("No output directory provided")
-    else:
-        if not output_path.exists():
-            output_path.mkdir(parents=True)
-            msg.good(f"Created output directory: {output_path}")
-        msg.info(f"Saving to output directory: {output_path}")
-    setup_gpu(use_gpu)
-    teacher = util.load_model(teacher_model)
-    with show_validation_error(student_config_path):
-        config = util.load_config(
-            student_config_path, overrides=overrides, interpolate=False
-        )
-    msg.divider("Initializing student pipeline")
-    with show_validation_error(student_config_path, hint_fill=False):
-        student = init_nlp_student(config, teacher, use_gpu=use_gpu)
-
-    msg.good("Initialized student pipeline")
-    msg.divider("Distilling student pipeline from teacher")
-    distill_nlp(
-        teacher,
-        student,
-        output_path,
-        use_gpu=use_gpu,
-        stdout=sys.stdout,
-        stderr=sys.stderr,
-    )

spacy/cli/download.py

@@ -1,3 +1,4 @@
+import shutil
 import sys
 from typing import Optional, Sequence
 from urllib.parse import urljoin
@@ -7,10 +8,9 @@ import typer
 from wasabi import msg
 
 from .. import about
+from ..errors import OLD_MODEL_SHORTCUTS
 from ..util import (
-    get_installed_models,
     get_minor_version,
-    get_package_version,
     is_in_interactive,
     is_in_jupyter,
     is_package,
@@ -28,8 +28,16 @@ def download_cli(
     # fmt: off
     ctx: typer.Context,
     model: str = Arg(..., help="Name of pipeline package to download"),
-    direct: bool = Opt(False, "--direct", "-d", "-D", help="Force direct download of name + version"),
-    sdist: bool = Opt(False, "--sdist", "-S", help="Download sdist (.tar.gz) archive instead of pre-built binary wheel"),
+    direct: bool = Opt(
+        False, "--direct", "-d", "-D", help="Force direct download of name + version"
+    ),
+    sdist: bool = Opt(
+        False,
+        "--sdist",
+        "-S",
+        help="Download sdist (.tar.gz) archive instead of pre-built binary wheel",
+    ),
+    url: str = Opt(None, "--url", "-U", help="Download from given url"),
     # fmt: on
 ):
     """
@@ -42,13 +50,14 @@ def download_cli(
     DOCS: https://spacy.io/api/cli#download
     AVAILABLE PACKAGES: https://spacy.io/models
     """
-    download(model, direct, sdist, *ctx.args)
+    download(model, direct, sdist, url, *ctx.args)
 
 
 def download(
     model: str,
     direct: bool = False,
     sdist: bool = False,
+    custom_url: Optional[str] = None,
     *pip_args,
 ) -> None:
     if (
@@ -77,20 +86,18 @@ def download(
         version = components[-1]
     else:
         model_name = model
+        if model in OLD_MODEL_SHORTCUTS:
+            msg.warn(
+                f"As of spaCy v3.0, shortcuts like '{model}' are deprecated. Please "
+                f"use the full pipeline package name '{OLD_MODEL_SHORTCUTS[model]}' instead."
+            )
+            model_name = OLD_MODEL_SHORTCUTS[model]
         compatibility = get_compatibility()
         version = get_version(model_name, compatibility)
 
-    # If we already have this version installed, skip downloading
-    installed = get_installed_models()
-    if model_name in installed:
-        installed_version = get_package_version(model_name)
-        if installed_version == version:
-            msg.warn(f"{model_name} v{version} already installed, skipping")
-            return
-
     filename = get_model_filename(model_name, version, sdist)
 
-    download_model(filename, pip_args)
+    download_model(filename, pip_args, custom_url)
     msg.good(
         "Download and installation successful",
         f"You can now load the package via spacy.load('{model_name}')",
@@ -162,12 +169,14 @@ def get_latest_version(model: str) -> str:
 
 
 def download_model(
-    filename: str, user_pip_args: Optional[Sequence[str]] = None
+    filename: str,
+    user_pip_args: Optional[Sequence[str]] = None,
+    custom_url: Optional[str] = None,
 ) -> None:
     # Construct the download URL carefully. We need to make sure we don't
     # allow relative paths or other shenanigans to trick us into download
     # from outside our own repo.
-    base_url = about.__download_url__
+    base_url = custom_url if custom_url else about.__download_url__
     # urljoin requires that the path ends with /, or the last path part will be dropped
     if not base_url.endswith("/"):
         base_url = about.__download_url__ + "/"
@@ -175,5 +184,19 @@ def download_model(
     if not download_url.startswith(about.__download_url__):
         raise ValueError(f"Download from {filename} rejected. Was it a relative path?")
     pip_args = list(user_pip_args) if user_pip_args is not None else []
-    cmd = [sys.executable, "-m", "pip", "install"] + pip_args + [download_url]
+    cmd = _get_pip_install_cmd() + pip_args + [download_url]
     run_command(cmd)
+
+
+def _get_pip_install_cmd() -> list:
+    if shutil.which("pip"):
+        return [sys.executable, "-m", "pip", "install"]
+    elif shutil.which("uv"):
+        return ["uv", "pip", "install"]
+    else:
+        msg.fail(
+            "No package installer found",
+            "spaCy requires either pip or uv to download models. "
+            "Please install one of them and try again.",
+            exits=1,
+        )

spacy/cli/evaluate.py

@@ -1,16 +1,15 @@
 import re
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional
 
 import srsly
 from thinc.api import fix_random_seed
 from wasabi import Printer
 
 from .. import displacy, util
-from ..scorer import Scorer
 from ..tokens import Doc
 from ..training import Corpus
-from ._util import Arg, Opt, app, benchmark_cli, import_code_paths, setup_gpu
+from ._util import Arg, Opt, app, benchmark_cli, import_code, setup_gpu
 
 
 @benchmark_cli.command(
@@ -20,15 +19,42 @@ from ._util import Arg, Opt, app, benchmark_cli, import_code_paths, setup_gpu
 def evaluate_cli(
     # fmt: off
     model: str = Arg(..., help="Model name or path"),
-    data_path: Path = Arg(..., help="Location of binary evaluation data in .spacy format", exists=True),
-    output: Optional[Path] = Opt(None, "--output", "-o", help="Output JSON file for metrics", dir_okay=False),
-    code_path: str = Opt("", "--code", "-c", help="Comma-separated paths to Python files with additional code (registered functions) to be imported"),
+    data_path: Path = Arg(
+        ..., help="Location of binary evaluation data in .spacy format", exists=True
+    ),
+    output: Optional[Path] = Opt(
+        None, "--output", "-o", help="Output JSON file for metrics", dir_okay=False
+    ),
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
     use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
-    gold_preproc: bool = Opt(False, "--gold-preproc", "-G", help="Use gold preprocessing"),
-    displacy_path: Optional[Path] = Opt(None, "--displacy-path", "-dp", help="Directory to output rendered parses as HTML", exists=True, file_okay=False),
-    displacy_limit: int = Opt(25, "--displacy-limit", "-dl", help="Limit of parses to render as HTML"),
-    per_component: bool = Opt(False, "--per-component", "-P", help="Return scores per component, only applicable when an output JSON file is specified."),
-    spans_key: str = Opt("sc", "--spans-key", "-sk", help="Spans key to use when evaluating Doc.spans"),
+    gold_preproc: bool = Opt(
+        False, "--gold-preproc", "-G", help="Use gold preprocessing"
+    ),
+    displacy_path: Optional[Path] = Opt(
+        None,
+        "--displacy-path",
+        "-dp",
+        help="Directory to output rendered parses as HTML",
+        exists=True,
+        file_okay=False,
+    ),
+    displacy_limit: int = Opt(
+        25, "--displacy-limit", "-dl", help="Limit of parses to render as HTML"
+    ),
+    per_component: bool = Opt(
+        False,
+        "--per-component",
+        "-P",
+        help="Return scores per component, only applicable when an output JSON file is specified.",
+    ),
+    spans_key: str = Opt(
+        "sc", "--spans-key", "-sk", help="Spans key to use when evaluating Doc.spans"
+    ),
     # fmt: on
 ):
     """
@@ -43,7 +69,7 @@ def evaluate_cli(
 
     DOCS: https://spacy.io/api/cli#benchmark-accuracy
     """
-    import_code_paths(code_path)
+    import_code(code_path)
     evaluate(
         model,
         data_path,
@@ -123,7 +149,7 @@ def evaluate(
                     if key == "speed":
                         results[metric] = f"{scores[key]:.0f}"
                     else:
-                        results[metric] = f"{scores[key]*100:.2f}"
+                        results[metric] = f"{scores[key] * 100:.2f}"
                 else:
                     results[metric] = "-"
                 data[re.sub(r"[\s/]", "_", key.lower())] = scores[key]

spacy/cli/find_function.py

@@ -11,7 +11,9 @@ from ._util import Arg, Opt, app
 def find_function_cli(
     # fmt: off
     func_name: str = Arg(..., help="Name of the registered function."),
-    registry_name: Optional[str] = Opt(None, "--registry", "-r", help="Name of the catalogue registry."),
+    registry_name: Optional[str] = Opt(
+        None, "--registry", "-r", help="Name of the catalogue registry."
+    ),
     # fmt: on
 ):
     """

spacy/cli/find_threshold.py

@@ -27,15 +27,39 @@ _DEFAULTS = {
 def find_threshold_cli(
     # fmt: off
     model: str = Arg(..., help="Model name or path"),
-    data_path: Path = Arg(..., help="Location of binary evaluation data in .spacy format", exists=True),
+    data_path: Path = Arg(
+        ..., help="Location of binary evaluation data in .spacy format", exists=True
+    ),
     pipe_name: str = Arg(..., help="Name of pipe to examine thresholds for"),
-    threshold_key: str = Arg(..., help="Key of threshold attribute in component's configuration"),
+    threshold_key: str = Arg(
+        ..., help="Key of threshold attribute in component's configuration"
+    ),
     scores_key: str = Arg(..., help="Metric to optimize"),
-    n_trials: int = Opt(_DEFAULTS["n_trials"], "--n_trials", "-n", help="Number of trials to determine optimal thresholds"),
-    code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
-    use_gpu: int = Opt(_DEFAULTS["use_gpu"], "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
-    gold_preproc: bool = Opt(_DEFAULTS["gold_preproc"], "--gold-preproc", "-G", help="Use gold preprocessing"),
-    verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
+    n_trials: int = Opt(
+        _DEFAULTS["n_trials"],
+        "--n_trials",
+        "-n",
+        help="Number of trials to determine optimal thresholds",
+    ),
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
+    use_gpu: int = Opt(
+        _DEFAULTS["use_gpu"], "--gpu-id", "-g", help="GPU ID or -1 for CPU"
+    ),
+    gold_preproc: bool = Opt(
+        _DEFAULTS["gold_preproc"], "--gold-preproc", "-G", help="Use gold preprocessing"
+    ),
+    verbose: bool = Opt(
+        False,
+        "--verbose",
+        "-V",
+        "-VV",
+        help="Display more information for debugging purposes",
+    ),
     # fmt: on
 ):
     """
@@ -157,9 +181,11 @@ def find_threshold(
                 exits=1,
             )
         return {
-            keys[0]: filter_config(config[keys[0]], keys[1:], full_key)
-            if len(keys) > 1
-            else config[keys[0]]
+            keys[0]: (
+                filter_config(config[keys[0]], keys[1:], full_key)
+                if len(keys) > 1
+                else config[keys[0]]
+            )
         }
 
     # Evaluate with varying threshold values.
@@ -181,10 +207,10 @@ def find_threshold(
             ),
         )
         if hasattr(pipe, "cfg"):
-            setattr(
-                nlp.get_pipe(pipe_name),
-                "cfg",
-                set_nested_item(getattr(pipe, "cfg"), config_keys, threshold),
+            nlp.get_pipe(pipe_name).cfg = set_nested_item(  # type: ignore[attr-defined]
+                pipe.cfg,
+                config_keys,
+                threshold,  # type: ignore[attr-defined]
             )
 
         eval_scores = nlp.evaluate(dev_dataset)
@@ -216,12 +242,14 @@ def find_threshold(
     if len(set(scores.values())) == 1:
         wasabi.msg.warn(
             title="All scores are identical. Verify that all settings are correct.",
-            text=""
-            if (
-                not isinstance(pipe, MultiLabel_TextCategorizer)
-                or scores_key in ("cats_macro_f", "cats_micro_f")
-            )
-            else "Use `cats_macro_f` or `cats_micro_f` when optimizing the threshold for `textcat_multilabel`.",
+            text=(
+                ""
+                if (
+                    not isinstance(pipe, MultiLabel_TextCategorizer)
+                    or scores_key in ("cats_macro_f", "cats_micro_f")
+                )
+                else "Use `cats_macro_f` or `cats_micro_f` when optimizing the threshold for `textcat_multilabel`."
+            ),
         )
 
     else:

spacy/cli/info.py

@@ -1,4 +1,3 @@
-import importlib.metadata
 import json
 import platform
 from pathlib import Path
@@ -8,6 +7,7 @@ import srsly
 from wasabi import MarkdownRenderer, Printer
 
 from .. import about, util
+from ..compat import importlib_metadata
 from ._util import Arg, Opt, app, string_to_list
 from .download import get_latest_version, get_model_filename
 
@@ -16,10 +16,24 @@ from .download import get_latest_version, get_model_filename
 def info_cli(
     # fmt: off
     model: Optional[str] = Arg(None, help="Optional loadable spaCy pipeline"),
-    markdown: bool = Opt(False, "--markdown", "-md", help="Generate Markdown for GitHub issues"),
-    silent: bool = Opt(False, "--silent", "-s", "-S", help="Don't print anything (just return)"),
-    exclude: str = Opt("labels", "--exclude", "-e", help="Comma-separated keys to exclude from the print-out"),
-    url: bool = Opt(False, "--url", "-u", help="Print the URL to download the most recent compatible version of the pipeline"),
+    markdown: bool = Opt(
+        False, "--markdown", "-md", help="Generate Markdown for GitHub issues"
+    ),
+    silent: bool = Opt(
+        False, "--silent", "-s", "-S", help="Don't print anything (just return)"
+    ),
+    exclude: str = Opt(
+        "labels",
+        "--exclude",
+        "-e",
+        help="Comma-separated keys to exclude from the print-out",
+    ),
+    url: bool = Opt(
+        False,
+        "--url",
+        "-u",
+        help="Print the URL to download the most recent compatible version of the pipeline",
+    ),
     # fmt: on
 ):
     """
@@ -137,7 +151,7 @@ def info_installed_model_url(model: str) -> Optional[str]:
     dist-info available.
     """
     try:
-        dist = importlib.metadata.distribution(model)
+        dist = importlib_metadata.distribution(model)
         text = dist.read_text("direct_url.json")
         if isinstance(text, str):
             data = json.loads(text)

spacy/cli/init_config.py

@@ -9,14 +9,13 @@ from thinc.api import Config
 from wasabi import Printer, diff_strings
 
 from .. import util
-from ..language import DEFAULT_CONFIG_DISTILL_PATH, DEFAULT_CONFIG_PRETRAIN_PATH
+from ..language import DEFAULT_CONFIG_PRETRAIN_PATH
 from ..schemas import RecommendationSchema
 from ..util import SimpleFrozenList
 from ._util import (
     COMMAND,
     Arg,
     Opt,
-    _handle_renamed_language_codes,
     import_code,
     init_cli,
     show_validation_error,
@@ -50,13 +49,44 @@ class InitValues:
 @init_cli.command("config")
 def init_config_cli(
     # fmt: off
-    output_file: Path = Arg(..., help="File to save the config to or - for stdout (will only output config and no additional logging info)", allow_dash=True),
-    lang: str = Opt(InitValues.lang, "--lang", "-l", help="Code of the language to use"),
-    pipeline: str = Opt(",".join(InitValues.pipeline), "--pipeline", "-p", help="Comma-separated names of trainable pipeline components to include (without 'tok2vec' or 'transformer')"),
-    optimize: Optimizations = Opt(InitValues.optimize, "--optimize", "-o", help="Whether to optimize for efficiency (faster inference, smaller model, lower memory consumption) or higher accuracy (potentially larger and slower model). This will impact the choice of architecture, pretrained weights and related hyperparameters."),
-    gpu: bool = Opt(InitValues.gpu, "--gpu", "-G", help="Whether the model can run on GPU. This will impact the choice of architecture, pretrained weights and related hyperparameters."),
-    pretraining: bool = Opt(InitValues.pretraining, "--pretraining", "-pt", help="Include config for pretraining (with 'spacy pretrain')"),
-    force_overwrite: bool = Opt(InitValues.force_overwrite, "--force", "-F", help="Force overwriting the output file"),
+    output_file: Path = Arg(
+        ...,
+        help="File to save the config to or - for stdout (will only output config and no additional logging info)",
+        allow_dash=True,
+    ),
+    lang: str = Opt(
+        InitValues.lang, "--lang", "-l", help="Two-letter code of the language to use"
+    ),
+    pipeline: str = Opt(
+        ",".join(InitValues.pipeline),
+        "--pipeline",
+        "-p",
+        help="Comma-separated names of trainable pipeline components to include (without 'tok2vec' or 'transformer')",
+    ),
+    optimize: Optimizations = Opt(
+        InitValues.optimize,
+        "--optimize",
+        "-o",
+        help="Whether to optimize for efficiency (faster inference, smaller model, lower memory consumption) or higher accuracy (potentially larger and slower model). This will impact the choice of architecture, pretrained weights and related hyperparameters.",
+    ),
+    gpu: bool = Opt(
+        InitValues.gpu,
+        "--gpu",
+        "-G",
+        help="Whether the model can run on GPU. This will impact the choice of architecture, pretrained weights and related hyperparameters.",
+    ),
+    pretraining: bool = Opt(
+        InitValues.pretraining,
+        "--pretraining",
+        "-pt",
+        help="Include config for pretraining (with 'spacy pretrain')",
+    ),
+    force_overwrite: bool = Opt(
+        InitValues.force_overwrite,
+        "--force",
+        "-F",
+        help="Force overwriting the output file",
+    ),
     # fmt: on
 ):
     """
@@ -89,12 +119,28 @@ def init_config_cli(
 @init_cli.command("fill-config")
 def init_fill_config_cli(
     # fmt: off
-    base_path: Path = Arg(..., help="Path to base config to fill", exists=True, dir_okay=False),
-    output_file: Path = Arg("-", help="Path to output .cfg file (or - for stdout)", allow_dash=True),
-    distillation: bool = Opt(False, "--distillation", "-dt", help="Include config for distillation (with 'spacy distill')"),
-    pretraining: bool = Opt(False, "--pretraining", "-pt", help="Include config for pretraining (with 'spacy pretrain')"),
-    diff: bool = Opt(False, "--diff", "-D", help="Print a visual diff highlighting the changes"),
-    code_path: Optional[Path] = Opt(None, "--code-path", "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
+    base_path: Path = Arg(
+        ..., help="Path to base config to fill", exists=True, dir_okay=False
+    ),
+    output_file: Path = Arg(
+        "-", help="Path to output .cfg file (or - for stdout)", allow_dash=True
+    ),
+    pretraining: bool = Opt(
+        False,
+        "--pretraining",
+        "-pt",
+        help="Include config for pretraining (with 'spacy pretrain')",
+    ),
+    diff: bool = Opt(
+        False, "--diff", "-D", help="Print a visual diff highlighting the changes"
+    ),
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code-path",
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
     # fmt: on
 ):
     """
@@ -107,20 +153,13 @@ def init_fill_config_cli(
     DOCS: https://spacy.io/api/cli#init-fill-config
     """
     import_code(code_path)
-    fill_config(
-        output_file,
-        base_path,
-        distillation=distillation,
-        pretraining=pretraining,
-        diff=diff,
-    )
+    fill_config(output_file, base_path, pretraining=pretraining, diff=diff)
 
 
 def fill_config(
     output_file: Path,
     base_path: Path,
     *,
-    distillation: bool = False,
     pretraining: bool = False,
     diff: bool = False,
     silent: bool = False,
@@ -139,9 +178,6 @@ def fill_config(
     # replaced with their actual config after loading, so we have to re-add them
     sourced = util.get_sourced_components(config)
     filled["components"].update(sourced)
-    if distillation:
-        distillation_config = util.load_config(DEFAULT_CONFIG_DISTILL_PATH)
-        filled = distillation_config.merge(filled)
     if pretraining:
         validate_config_for_pretrain(filled, msg)
         pretrain_config = util.load_config(DEFAULT_CONFIG_PRETRAIN_PATH)
@@ -177,14 +213,10 @@ def init_config(
     msg = Printer(no_print=silent)
     with TEMPLATE_PATH.open("r") as f:
         template = Template(f.read())
-
-    # Throw error for renamed language codes in v4
-    _handle_renamed_language_codes(lang)
-
     # Filter out duplicates since tok2vec and transformer are added by template
     pipeline = [pipe for pipe in pipeline if pipe not in ("tok2vec", "transformer")]
     defaults = RECOMMENDATIONS["__default__"]
-    reco = RecommendationSchema(**RECOMMENDATIONS.get(lang, defaults)).dict()
+    reco = RecommendationSchema(**RECOMMENDATIONS.get(lang, defaults)).model_dump()
     variables = {
         "lang": lang,
         "components": pipeline,
@@ -211,9 +243,11 @@ def init_config(
         "Pipeline": ", ".join(pipeline),
         "Optimize for": optimize,
         "Hardware": variables["hardware"].upper(),
-        "Transformer": template_vars.transformer.get("name")  # type: ignore[attr-defined]
-        if template_vars.use_transformer  # type: ignore[attr-defined]
-        else None,
+        "Transformer": (
+            template_vars.transformer.get("name")  # type: ignore[attr-defined]
+            if template_vars.use_transformer  # type: ignore[attr-defined]
+            else None
+        ),
     }
     msg.info("Generated config template specific for your use case")
     for label, value in use_case.items():

spacy/cli/init_pipeline.py

@@ -12,7 +12,6 @@ from ..training.initialize import convert_vectors, init_nlp
 from ._util import (
     Arg,
     Opt,
-    _handle_renamed_language_codes,
     import_code,
     init_cli,
     parse_config_overrides,
@@ -27,23 +26,50 @@ def init_vectors_cli(
     lang: str = Arg(..., help="The language of the nlp object to create"),
     vectors_loc: Path = Arg(..., help="Vectors file in Word2Vec format", exists=True),
     output_dir: Path = Arg(..., help="Pipeline output directory"),
-    prune: int = Opt(-1, "--prune", "-p", help="Optional number of vectors to prune to"),
-    truncate: int = Opt(0, "--truncate", "-t", help="Optional number of vectors to truncate to when reading in vectors file"),
+    prune: int = Opt(
+        -1, "--prune", "-p", help="Optional number of vectors to prune to"
+    ),
+    truncate: int = Opt(
+        0,
+        "--truncate",
+        "-t",
+        help="Optional number of vectors to truncate to when reading in vectors file",
+    ),
     mode: str = Opt("default", "--mode", "-m", help="Vectors mode: default or floret"),
-    verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
-    jsonl_loc: Optional[Path] = Opt(None, "--lexemes-jsonl", "-j", help="Location of JSONL-formatted attributes file", hidden=True),
-    attr: str = Opt("ORTH", "--attr", "-a", help="Optional token attribute to use for vectors, e.g. LOWER or NORM"),
+    name: Optional[str] = Opt(
+        None,
+        "--name",
+        "-n",
+        help="Optional name for the word vectors, e.g. en_core_web_lg.vectors",
+    ),
+    verbose: bool = Opt(
+        False,
+        "--verbose",
+        "-V",
+        "-VV",
+        help="Display more information for debugging purposes",
+    ),
+    jsonl_loc: Optional[Path] = Opt(
+        None,
+        "--lexemes-jsonl",
+        "-j",
+        help="Location of JSONL-formatted attributes file",
+        hidden=True,
+    ),
+    attr: str = Opt(
+        "ORTH",
+        "--attr",
+        "-a",
+        help="Optional token attribute to use for vectors, e.g. LOWER or NORM",
+    ),
     # fmt: on
 ):
     """Convert word vectors for use with spaCy. Will export an nlp object that
     you can use in the [initialize] block of your config to initialize
     a model with vectors.
     """
-    util.logger.setLevel(logging.DEBUG if verbose else logging.INFO)
-
-    # Throw error for renamed language codes in v4
-    _handle_renamed_language_codes(lang)
-
+    if verbose:
+        util.logger.setLevel(logging.DEBUG)
     msg.info(f"Creating blank nlp object for language '{lang}'")
     nlp = util.get_lang_class(lang)()
     if jsonl_loc is not None:
@@ -53,6 +79,7 @@ def init_vectors_cli(
         vectors_loc,
         truncate=truncate,
         prune=prune,
+        name=name,
         mode=mode,
         attr=attr,
     )
@@ -83,11 +110,24 @@ def update_lexemes(nlp: Language, jsonl_loc: Path) -> None:
 def init_pipeline_cli(
     # fmt: off
     ctx: typer.Context,  # This is only used to read additional arguments
-    config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
+    config_path: Path = Arg(
+        ..., help="Path to config file", exists=True, allow_dash=True
+    ),
     output_path: Path = Arg(..., help="Output directory for the prepared data"),
-    code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
-    verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
-    use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU")
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
+    verbose: bool = Opt(
+        False,
+        "--verbose",
+        "-V",
+        "-VV",
+        help="Display more information for debugging purposes",
+    ),
+    use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
     # fmt: on
 ):
     if verbose:
@@ -110,11 +150,24 @@ def init_pipeline_cli(
 def init_labels_cli(
     # fmt: off
     ctx: typer.Context,  # This is only used to read additional arguments
-    config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
+    config_path: Path = Arg(
+        ..., help="Path to config file", exists=True, allow_dash=True
+    ),
     output_path: Path = Arg(..., help="Output directory for the labels"),
-    code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
-    verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
-    use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU")
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
+    verbose: bool = Opt(
+        False,
+        "--verbose",
+        "-V",
+        "-VV",
+        help="Display more information for debugging purposes",
+    ),
+    use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
     # fmt: on
 ):
     """Generate JSON files for the labels in the data. This helps speed up the

spacy/cli/package.py

@@ -1,4 +1,3 @@
-import importlib.metadata
 import os
 import re
 import shutil
@@ -14,6 +13,7 @@ from thinc.api import Config
 from wasabi import MarkdownRenderer, Printer, get_raw_input
 
 from .. import about, util
+from ..compat import importlib_metadata
 from ..schemas import ModelMetaSchema, validate
 from ._util import SDIST_SUFFIX, WHEEL_SUFFIX, Arg, Opt, app, string_to_list
 
@@ -21,15 +21,56 @@ from ._util import SDIST_SUFFIX, WHEEL_SUFFIX, Arg, Opt, app, string_to_list
 @app.command("package")
 def package_cli(
     # fmt: off
-    input_dir: Path = Arg(..., help="Directory with pipeline data", exists=True, file_okay=False),
-    output_dir: Path = Arg(..., help="Output parent directory", exists=True, file_okay=False),
-    code_paths: str = Opt("", "--code", "-c", help="Comma-separated paths to Python files with additional code (registered functions) to be included in the package"),
-    meta_path: Optional[Path] = Opt(None, "--meta-path", "--meta", "-m", help="Path to meta.json", exists=True, dir_okay=False),
-    create_meta: bool = Opt(False, "--create-meta", "-C", help="Create meta.json, even if one exists"),
-    name: Optional[str] = Opt(None, "--name", "-n", help="Package name to override meta"),
-    version: Optional[str] = Opt(None, "--version", "-v", help="Package version to override meta"),
-    build: str = Opt("sdist", "--build", "-b", help="Comma-separated formats to build: sdist and/or wheel, or none."),
-    force: bool = Opt(False, "--force", "-f", "-F", help="Force overwriting existing data in output directory"),
+    input_dir: Path = Arg(
+        ..., help="Directory with pipeline data", exists=True, file_okay=False
+    ),
+    output_dir: Path = Arg(
+        ..., help="Output parent directory", exists=True, file_okay=False
+    ),
+    code_paths: str = Opt(
+        "",
+        "--code",
+        "-c",
+        help="Comma-separated paths to Python file with additional code (registered functions) to be included in the package",
+    ),
+    meta_path: Optional[Path] = Opt(
+        None,
+        "--meta-path",
+        "--meta",
+        "-m",
+        help="Path to meta.json",
+        exists=True,
+        dir_okay=False,
+    ),
+    create_meta: bool = Opt(
+        False, "--create-meta", "-C", help="Create meta.json, even if one exists"
+    ),
+    name: Optional[str] = Opt(
+        None, "--name", "-n", help="Package name to override meta"
+    ),
+    version: Optional[str] = Opt(
+        None, "--version", "-v", help="Package version to override meta"
+    ),
+    build: str = Opt(
+        "sdist",
+        "--build",
+        "-b",
+        help="Comma-separated formats to build: sdist and/or wheel, or none.",
+    ),
+    force: bool = Opt(
+        False,
+        "--force",
+        "-f",
+        "-F",
+        help="Force overwriting existing data in output directory",
+    ),
+    require_parent: bool = Opt(
+        True,
+        "--require-parent/--no-require-parent",
+        "-R",
+        "-R",
+        help="Include the parent package (e.g. spacy) in the requirements",
+    ),
     # fmt: on
 ):
     """
@@ -60,6 +101,7 @@ def package_cli(
         create_sdist=create_sdist,
         create_wheel=create_wheel,
         force=force,
+        require_parent=require_parent,
         silent=False,
     )
 
@@ -74,6 +116,7 @@ def package(
     create_meta: bool = False,
     create_sdist: bool = True,
     create_wheel: bool = False,
+    require_parent: bool = False,
     force: bool = False,
     silent: bool = True,
 ) -> None:
@@ -113,7 +156,7 @@ def package(
     if not meta_path.exists() or not meta_path.is_file():
         msg.fail("Can't load pipeline meta.json", meta_path, exits=1)
     meta = srsly.read_json(meta_path)
-    meta = get_meta(input_dir, meta)
+    meta = get_meta(input_dir, meta, require_parent=require_parent)
     if meta["requirements"]:
         msg.good(
             f"Including {len(meta['requirements'])} package requirement(s) from "
@@ -186,6 +229,7 @@ def package(
         imports.append(code_path.stem)
         shutil.copy(str(code_path), str(package_path))
     create_file(main_path / "meta.json", srsly.json_dumps(meta, indent=2))
+
     create_file(main_path / "setup.py", TEMPLATE_SETUP)
     create_file(main_path / "MANIFEST.in", TEMPLATE_MANIFEST)
     init_py = TEMPLATE_INIT.format(
@@ -250,9 +294,9 @@ def has_build() -> bool:
     # in an editable install), so an import check is not sufficient; instead
     # check that there is a package version
     try:
-        importlib.metadata.version("build")
+        importlib_metadata.version("build")
         return True
-    except importlib.metadata.PackageNotFoundError:  # type: ignore[attr-defined]
+    except importlib_metadata.PackageNotFoundError:  # type: ignore[attr-defined]
         return False
 
 
@@ -302,6 +346,8 @@ def get_third_party_dependencies(
                 modules.add(func_info["module"].split(".")[0])  # type: ignore[union-attr]
     dependencies = []
     for module_name in modules:
+        if module_name == about.__title__:
+            continue
         if module_name in distributions:
             dist = distributions.get(module_name)
             if dist:
@@ -332,7 +378,9 @@ def create_file(file_path: Path, contents: str) -> None:
 
 
 def get_meta(
-    model_path: Union[str, Path], existing_meta: Dict[str, Any]
+    model_path: Union[str, Path],
+    existing_meta: Dict[str, Any],
+    require_parent: bool = False,
 ) -> Dict[str, Any]:
     meta: Dict[str, Any] = {
         "lang": "en",
@@ -352,6 +400,7 @@ def get_meta(
         "width": nlp.vocab.vectors_length,
         "vectors": len(nlp.vocab.vectors),
         "keys": nlp.vocab.vectors.n_keys,
+        "name": nlp.vocab.vectors.name,
     }
     if about.__title__ != "spacy":
         meta["parent_package"] = about.__title__
@@ -360,6 +409,8 @@ def get_meta(
     existing_reqs = [util.split_requirement(req)[0] for req in meta["requirements"]]
     reqs = get_third_party_dependencies(nlp.config, exclude=existing_reqs)
     meta["requirements"].extend(reqs)
+    if require_parent and about.__title__ not in meta["requirements"]:
+        meta["requirements"].append(about.__title__ + meta["spacy_version"])
     return meta
 
 
@@ -399,7 +450,7 @@ def generate_readme(meta: Dict[str, Any]) -> str:
     pipeline = ", ".join([md.code(p) for p in meta.get("pipeline", [])])
     components = ", ".join([md.code(p) for p in meta.get("components", [])])
     vecs = meta.get("vectors", {})
-    vectors = f"{vecs.get('keys', 0)} keys, {vecs.get('vectors', 0)} unique vectors ({ vecs.get('width', 0)} dimensions)"
+    vectors = f"{vecs.get('keys', 0)} keys, {vecs.get('vectors', 0)} unique vectors ({vecs.get('width', 0)} dimensions)"
     author = meta.get("author") or "n/a"
     notes = meta.get("notes", "")
     license_name = meta.get("license")
@@ -458,7 +509,7 @@ def _format_accuracy(data: Dict[str, Any], exclude: List[str] = ["speed"]) -> st
     md = MarkdownRenderer()
     scalars = [(k, v) for k, v in data.items() if isinstance(v, (int, float))]
     scores = [
-        (md.code(acc.upper()), f"{score*100:.2f}")
+        (md.code(acc.upper()), f"{score * 100:.2f}")
         for acc, score in scalars
         if acc not in exclude
     ]
@@ -477,9 +528,7 @@ def _format_label_scheme(data: Dict[str, Any]) -> str:
         if not labels:
             continue
         col1 = md.bold(md.code(pipe))
-        col2 = ", ".join(
-            [md.code(str(label).replace("|", "\\|")) for label in labels]
-        )  # noqa: W605
+        col2 = ", ".join([md.code(str(label).replace("|", "\\|")) for label in labels])  # noqa: W605
         label_data.append((col1, col2))
         n_labels += len(labels)
         n_pipes += 1
@@ -534,8 +583,11 @@ def list_files(data_dir):
 
 
 def list_requirements(meta):
-    parent_package = meta.get('parent_package', 'spacy')
-    requirements = [parent_package + meta['spacy_version']]
+    # Up to version 3.7, we included the parent package
+    # in requirements by default. This behaviour is removed
+    # in 3.8, with a setting to include the parent package in
+    # the requirements list in the meta if desired.
+    requirements = []
     if 'setup_requires' in meta:
         requirements += meta['setup_requires']
     if 'requirements' in meta:

spacy/cli/pretrain.py

@@ -11,7 +11,7 @@ from ._util import (
     Arg,
     Opt,
     app,
-    import_code_paths,
+    import_code,
     parse_config_overrides,
     setup_gpu,
     show_validation_error,
@@ -25,13 +25,32 @@ from ._util import (
 def pretrain_cli(
     # fmt: off
     ctx: typer.Context,  # This is only used to read additional arguments
-    config_path: Path = Arg(..., help="Path to config file", exists=True, dir_okay=False, allow_dash=True),
+    config_path: Path = Arg(
+        ..., help="Path to config file", exists=True, dir_okay=False, allow_dash=True
+    ),
     output_dir: Path = Arg(..., help="Directory to write weights to on each epoch"),
-    code_path: str = Opt("", "--code", "-c", help="Comma-separated paths to Python files with additional code (registered functions) to be imported"),
-    resume_path: Optional[Path] = Opt(None, "--resume-path", "-r", help="Path to pretrained weights from which to resume pretraining"),
-    epoch_resume: Optional[int] = Opt(None, "--epoch-resume", "-er", help="The epoch to resume counting from when using --resume-path. Prevents unintended overwriting of existing weight files."),
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
+    resume_path: Optional[Path] = Opt(
+        None,
+        "--resume-path",
+        "-r",
+        help="Path to pretrained weights from which to resume pretraining",
+    ),
+    epoch_resume: Optional[int] = Opt(
+        None,
+        "--epoch-resume",
+        "-er",
+        help="The epoch to resume counting from when using --resume-path. Prevents unintended overwriting of existing weight files.",
+    ),
     use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
-    skip_last: bool = Opt(False, "--skip-last", "-L", help="Skip saving model-last.bin"),
+    skip_last: bool = Opt(
+        False, "--skip-last", "-L", help="Skip saving model-last.bin"
+    ),
     # fmt: on
 ):
     """
@@ -56,7 +75,7 @@ def pretrain_cli(
     DOCS: https://spacy.io/api/cli#pretrain
     """
     config_overrides = parse_config_overrides(ctx.args)
-    import_code_paths(code_path)
+    import_code(code_path)
     verify_cli_args(config_path, output_dir, resume_path, epoch_resume)
     setup_gpu(use_gpu)
     msg.info(f"Loading config from: {config_path}")

spacy/cli/profile.py

@@ -21,8 +21,15 @@ def profile_cli(
     # fmt: off
     ctx: typer.Context,  # This is only used to read current calling context
     model: str = Arg(..., help="Trained pipeline to load"),
-    inputs: Optional[Path] = Arg(None, help="Location of input file. '-' for stdin.", exists=True, allow_dash=True),
-    n_texts: int = Opt(10000, "--n-texts", "-n", help="Maximum number of texts to use if available"),
+    inputs: Optional[Path] = Arg(
+        None,
+        help="Location of input file. '-' for stdin.",
+        exists=True,
+        allow_dash=True,
+    ),
+    n_texts: int = Opt(
+        10000, "--n-texts", "-n", help="Maximum number of texts to use if available"
+    ),
     # fmt: on
 ):
     """

spacy/cli/templates/quickstart_training.jinja

@@ -238,7 +238,7 @@ grad_factor = 1.0
 {% if "entity_linker" in components -%}
 [components.entity_linker]
 factory = "entity_linker"
-get_candidates = {"@misc":"spacy.CandidateGenerator.v2"}
+get_candidates = {"@misc":"spacy.CandidateGenerator.v1"}
 incl_context = true
 incl_prior = true
 
@@ -517,7 +517,7 @@ width = ${components.tok2vec.model.encode.width}
 {% if "entity_linker" in components -%}
 [components.entity_linker]
 factory = "entity_linker"
-get_candidates = {"@misc":"spacy.CandidateGenerator.v2"}
+get_candidates = {"@misc":"spacy.CandidateGenerator.v1"}
 incl_context = true
 incl_prior = true
 

spacy/cli/train.py

@@ -13,7 +13,7 @@ from ._util import (
     Arg,
     Opt,
     app,
-    import_code_paths,
+    import_code,
     parse_config_overrides,
     setup_gpu,
     show_validation_error,
@@ -26,11 +26,30 @@ from ._util import (
 def train_cli(
     # fmt: off
     ctx: typer.Context,  # This is only used to read additional arguments
-    config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
-    output_path: Optional[Path] = Opt(None, "--output", "--output-path", "-o", help="Output directory to store trained pipeline in"),
-    code_path: str = Opt("", "--code", "-c", help="Comma-separated paths to Python files with additional code (registered functions) to be imported"),
-    verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
-    use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU")
+    config_path: Path = Arg(
+        ..., help="Path to config file", exists=True, allow_dash=True
+    ),
+    output_path: Optional[Path] = Opt(
+        None,
+        "--output",
+        "--output-path",
+        "-o",
+        help="Output directory to store trained pipeline in",
+    ),
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
+    verbose: bool = Opt(
+        False,
+        "--verbose",
+        "-V",
+        "-VV",
+        help="Display more information for debugging purposes",
+    ),
+    use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
     # fmt: on
 ):
     """
@@ -50,7 +69,7 @@ def train_cli(
     if verbose:
         util.logger.setLevel(logging.DEBUG)
     overrides = parse_config_overrides(ctx.args)
-    import_code_paths(code_path)
+    import_code(code_path)
     train(config_path, output_path, use_gpu=use_gpu, overrides=overrides)
 
 

spacy/compat.py

@@ -1,4 +1,5 @@
 """Helpers for Python and platform compatibility."""
+
 import sys
 
 from thinc.util import copy_array
@@ -23,6 +24,21 @@ try:
 except ImportError:
     cupy = None
 
+if sys.version_info[:2] >= (3, 8):  # Python 3.8+
+    from typing import Literal, Protocol, runtime_checkable
+else:
+    from typing_extensions import Literal, Protocol, runtime_checkable  # noqa: F401
+
+# Important note: The importlib_metadata "backport" includes functionality
+# that's not part of the built-in importlib.metadata. We should treat this
+# import like the built-in and only use what's available there.
+try:  # Python 3.8+
+    import importlib.metadata as importlib_metadata
+except ImportError:
+    from catalogue import (  # type: ignore[no-redef]
+        _importlib_metadata as importlib_metadata,  # noqa: F401
+    )
+
 from thinc.api import Optimizer  # noqa: F401
 
 pickle = pickle

spacy/default_config_distillation.cfg

@@ -1,34 +0,0 @@
-[paths]
-raw_text = null
-
-[distillation]
-corpus = "corpora.distillation"
-dropout = 0.1
-max_epochs = 1
-max_steps = 0
-student_to_teacher = {}
-
-[distillation.batcher]
-@batchers = "spacy.batch_by_words.v1"
-size = 3000
-discard_oversize = false
-tolerance = 0.2
-
-[distillation.optimizer]
-@optimizers = "Adam.v1"
-beta1 = 0.9
-beta2 = 0.999
-L2_is_weight_decay = true
-L2 = 0.01
-grad_clip = 1.0
-use_averages = true
-eps = 1e-8
-learn_rate = 1e-4
-
-[corpora]
-
-[corpora.distillation]
-@readers = "spacy.PlainTextCorpus.v1"
-path = ${paths.raw_text}
-min_length = 0
-max_length = 0

spacy/displacy/__init__.py

@@ -4,6 +4,7 @@ spaCy's built in visualization suite for dependencies and named entities.
 DOCS: https://spacy.io/api/top-level#displacy
 USAGE: https://spacy.io/usage/visualizers
 """
+
 import warnings
 from typing import Any, Callable, Dict, Iterable, Optional, Union
 
@@ -66,7 +67,7 @@ def render(
     if jupyter or (jupyter is None and is_in_jupyter()):
         # return HTML rendered by IPython display()
         # See #4840 for details on span wrapper to disable mathjax
-        from IPython.core.display import HTML, display
+        from IPython.display import HTML, display
 
         return display(HTML('<span class="tex2jax_ignore">{}</span>'.format(html)))
     return html

spacy/errors.py

@@ -1,7 +1,6 @@
 import warnings
-from typing import Literal
 
-from . import about
+from .compat import Literal
 
 
 class ErrorsWithCodes(type):
@@ -84,7 +83,7 @@ class Warnings(metaclass=ErrorsWithCodes):
             "ignoring the duplicate entry.")
     W021 = ("Unexpected hash collision in PhraseMatcher. Matches may be "
             "incorrect. Modify PhraseMatcher._terminal_hash to fix.")
-    W024 = ("Entity '{entity}' - alias '{alias}' combination already exists in "
+    W024 = ("Entity '{entity}' - Alias '{alias}' combination already exists in "
             "the Knowledge Base.")
     W026 = ("Unable to set all sentence boundaries from dependency parses. If "
             "you are constructing a parse tree incrementally by setting "
@@ -105,14 +104,13 @@ class Warnings(metaclass=ErrorsWithCodes):
             "table. This may degrade the performance of the model to some "
             "degree. If this is intentional or the language you're using "
             "doesn't have a normalization table, please ignore this warning. "
-            "If this is surprising, make sure you are loading the table in "
-            "your config. The languages with lexeme normalization tables are "
-            "currently: {langs}\n\nAn example of how to load a table in "
-            "your config :\n\n"
+            "If this is surprising, make sure you have the spacy-lookups-data "
+            "package installed and load the table in your config. The "
+            "languages with lexeme normalization tables are currently: "
+            "{langs}\n\nLoad the table in your config with:\n\n"
             "[initialize.lookups]\n"
-            "@misc = \"spacy.LookupsDataLoaderFromURL.v1\"\n"
+            "@misc = \"spacy.LookupsDataLoader.v1\"\n"
             "lang = ${{nlp.lang}}\n"
-             f'url = "{about.__lookups_url__}"\n'
             "tables = [\"lexeme_norm\"]\n")
     W035 = ("Discarding subpattern '{pattern}' due to an unrecognized "
             "attribute or operator.")
@@ -134,6 +132,13 @@ class Warnings(metaclass=ErrorsWithCodes):
             "and make it independent. For example, `replace_listeners = "
             "[\"model.tok2vec\"]` See the documentation for details: "
             "https://spacy.io/usage/training#config-components-listeners")
+    W088 = ("The pipeline component {name} implements a `begin_training` "
+            "method, which won't be called by spaCy. As of v3.0, `begin_training` "
+            "has been renamed to `initialize`, so you likely want to rename the "
+            "component method. See the documentation for details: "
+            "https://spacy.io/api/language#initialize")
+    W089 = ("As of spaCy v3.0, the `nlp.begin_training` method has been renamed "
+            "to `nlp.initialize`.")
     W090 = ("Could not locate any {format} files in path '{path}'.")
     W091 = ("Could not clean/remove the temp directory at {dir}: {msg}.")
     W092 = ("Ignoring annotations for sentence starts, as dependency heads are set.")
@@ -217,11 +222,6 @@ class Warnings(metaclass=ErrorsWithCodes):
     W126 = ("These keys are unsupported: {unsupported}")
     W127 = ("Not all `Language.pipe` worker processes completed successfully")
 
-    # v4 warning strings
-    W401 = ("`incl_prior is True`, but the selected knowledge base type {kb_type} doesn't support prior probability "
-            "lookups so this setting will be ignored. If your KB does support prior probability lookups, make sure "
-            "to return `True` in `.supports_prior_probs`.")
-
 
 class Errors(metaclass=ErrorsWithCodes):
     E001 = ("No component '{name}' found in pipeline. Available names: {opts}")
@@ -256,7 +256,9 @@ class Errors(metaclass=ErrorsWithCodes):
             "https://spacy.io/usage/models")
     E011 = ("Unknown operator: '{op}'. Options: {opts}")
     E012 = ("Cannot add pattern for zero tokens to matcher.\nKey: {key}")
-    E017 = ("Can only add 'str' inputs to StringStore. Got type: {value_type}")
+    E016 = ("MultitaskObjective target should be function or one of: dep, "
+            "tag, ent, dep_tag_offset, ent_tag.")
+    E017 = ("Can only add unicode or bytes. Got type: {value_type}")
     E018 = ("Can't retrieve string for hash '{hash_value}'. This usually "
             "refers to an issue with the `Vocab` or `StringStore`.")
     E019 = ("Can't create transition with unknown action ID: {action}. Action "
@@ -468,13 +470,13 @@ class Errors(metaclass=ErrorsWithCodes):
             "same, but found '{nlp}' and '{vocab}' respectively.")
     E152 = ("The attribute {attr} is not supported for token patterns. "
             "Please use the option `validate=True` with the Matcher, PhraseMatcher, "
-            "SpanRuler or AttributeRuler for more details.")
+            "EntityRuler or AttributeRuler for more details.")
     E153 = ("The value type {vtype} is not supported for token patterns. "
             "Please use the option validate=True with Matcher, PhraseMatcher, "
-            "SpanRuler or AttributeRuler for more details.")
+            "EntityRuler or AttributeRuler for more details.")
     E154 = ("One of the attributes or values is not supported for token "
             "patterns. Please use the option `validate=True` with the Matcher, "
-            "PhraseMatcher, or SpanRuler for more details.")
+            "PhraseMatcher, or EntityRuler for more details.")
     E155 = ("The pipeline needs to include a {pipe} in order to use "
             "Matcher or PhraseMatcher with the attribute {attr}. "
             "Try using `nlp()` instead of `nlp.make_doc()` or `list(nlp.pipe())` "
@@ -498,7 +500,7 @@ class Errors(metaclass=ErrorsWithCodes):
             "Current DocBin: {current}\nOther DocBin: {other}")
     E169 = ("Can't find module: {module}")
     E170 = ("Cannot apply transition {name}: invalid for the current state.")
-    E171 = ("{name}.add received invalid 'on_match' callback argument: expected "
+    E171 = ("Matcher.add received invalid 'on_match' callback argument: expected "
             "callable or None, but got: {arg_type}")
     E175 = ("Can't remove rule for unknown match pattern ID: {key}")
     E176 = ("Alias '{alias}' is not defined in the Knowledge Base.")
@@ -737,6 +739,13 @@ class Errors(metaclass=ErrorsWithCodes):
             "method in component '{name}'. If you want to use this "
             "method, make sure it's overwritten on the subclass.")
     E940 = ("Found NaN values in scores.")
+    E941 = ("Can't find model '{name}'. It looks like you're trying to load a "
+            "model from a shortcut, which is obsolete as of spaCy v3.0. To "
+            "load the model, use its full name instead:\n\n"
+            "nlp = spacy.load(\"{full}\")\n\nFor more details on the available "
+            "models, see the models directory: https://spacy.io/models and if "
+            "you want to create a blank model, use spacy.blank: "
+            "nlp = spacy.blank(\"{name}\")")
     E942 = ("Executing `after_{name}` callback failed. Expected the function to "
             "return an initialized nlp object but got: {value}. Maybe "
             "you forgot to return the modified object in your function?")
@@ -750,7 +759,7 @@ class Errors(metaclass=ErrorsWithCodes):
             "loaded nlp object, but got: {source}")
     E947 = ("`Matcher.add` received invalid `greedy` argument: expected "
             "a string value from {expected} but got: '{arg}'")
-    E948 = ("`{name}.add` received invalid 'patterns' argument: expected "
+    E948 = ("`Matcher.add` received invalid 'patterns' argument: expected "
             "a list, but got: {arg_type}")
     E949 = ("Unable to align tokens for the predicted and reference docs. It "
             "is only possible to align the docs when both texts are the same "
@@ -924,6 +933,8 @@ class Errors(metaclass=ErrorsWithCodes):
     E1021 = ("`pos` value \"{pp}\" is not a valid Universal Dependencies tag. "
              "Non-UD tags should use the `tag` property.")
     E1022 = ("Words must be of type str or int, but input is of type '{wtype}'")
+    E1023 = ("Couldn't read EntityRuler from the {path}. This file doesn't "
+             "exist.")
     E1024 = ("A pattern with {attr_type} '{label}' is not present in "
              "'{component}' patterns.")
     E1025 = ("Cannot intify the value '{value}' as an IOB string. The only "
@@ -934,7 +945,7 @@ class Errors(metaclass=ErrorsWithCodes):
     E1029 = ("Edit tree cannot be applied to form.")
     E1030 = ("Edit tree identifier out of range.")
     E1031 = ("Could not find gold transition - see logs above.")
-    E1032 = ("Span {var} {value} is out of bounds for {obj} with length {length}.")
+    E1032 = ("`{var}` should not be {forbidden}, but received {value}.")
     E1033 = ("Dimension {name} invalid -- only nO, nF, nP")
     E1034 = ("Node index {i} out of bounds ({length})")
     E1035 = ("Token index {i} out of bounds ({length})")
@@ -951,6 +962,7 @@ class Errors(metaclass=ErrorsWithCodes):
              "case pass an empty list for the previously not specified argument to avoid this error.")
     E1043 = ("Expected None or a value in range [{range_start}, {range_end}] for entity linker threshold, but got "
              "{value}.")
+    E1044 = ("Expected `candidates_batch_size` to be >= 1, but got: {value}")
     E1045 = ("Encountered {parent} subclass without `{parent}.{method}` "
              "method in '{name}'. If you want to use this method, make "
              "sure it's overwritten on the subclass.")
@@ -977,35 +989,15 @@ class Errors(metaclass=ErrorsWithCodes):
              "reduction. Please enable one of `use_reduce_first`, "
              "`use_reduce_last`, `use_reduce_max` or `use_reduce_mean`.")
 
-    # v4 error strings
-    E4000 = ("Expected a Doc as input, but got: '{type}'")
-    E4001 = ("Expected input to be one of the following types: ({expected_types}), "
-             "but got '{received_type}'")
-    E4002 = ("Pipe '{name}' requires a teacher pipe for distillation.")
-    E4003 = ("Training examples for distillation must have the exact same tokens in the "
-             "reference and predicted docs.")
-    E4004 = ("Backprop is not supported when is_train is not set.")
-    E4005 = ("EntityLinker_v1 is not supported in spaCy v4. Update your configuration.")
-    E4006 = ("Expected `entity_id` to be of type {exp_type}, but is of type {found_type}.")
-    E4007 = ("Span {var} {value} must be {op} Span {existing_var} "
-             "{existing_value}.")
-    E4008 = ("Span {pos}_char {value} does not correspond to a token {pos}.")
-    E4009 = ("The '{attr}' parameter should be 'None' or 'True', but found '{value}'.")
-    E4010 = ("Required lemmatizer table(s) {missing_tables} not found in "
-             "[initialize] or in registered lookups (spacy-lookups-data). An "
-             "example for how to load lemmatizer tables in [initialize]:\n\n"
-             "[initialize.components]\n\n"
-             "[initialize.components.{pipe_name}]\n\n"
-             "[initialize.components.{pipe_name}.lookups]\n"
-             '@misc = "spacy.LookupsDataLoaderFromURL.v1"\n'
-             "lang = ${{nlp.lang}}\n"
-             f'url = "{about.__lookups_url__}"\n'
-             "tables = {tables}\n"
-             "# or required tables only: tables = {required_tables}\n")
-    E4011 = ("Server error ({status_code}), couldn't fetch {url}")
 
+# Deprecated model shortcuts, only used in errors and warnings
+OLD_MODEL_SHORTCUTS = {
+    "en": "en_core_web_sm", "de": "de_core_news_sm", "es": "es_core_news_sm",
+    "pt": "pt_core_news_sm", "fr": "fr_core_news_sm", "it": "it_core_news_sm",
+    "nl": "nl_core_news_sm", "el": "el_core_news_sm", "nb": "nb_core_news_sm",
+    "lt": "lt_core_news_sm", "xx": "xx_ent_wiki_sm"
+}
 
-RENAMED_LANGUAGE_CODES = {"xx": "mul", "is": "isl"}
 
 # fmt: on
 

spacy/kb/__init__.py

@@ -1,10 +1,11 @@
-from .candidate import Candidate, InMemoryCandidate
+from .candidate import Candidate, get_candidates, get_candidates_batch
 from .kb import KnowledgeBase
 from .kb_in_memory import InMemoryLookupKB
 
 __all__ = [
     "Candidate",
     "KnowledgeBase",
-    "InMemoryCandidate",
     "InMemoryLookupKB",
+    "get_candidates",
+    "get_candidates_batch",
 ]

spacy/kb/candidate.pxd

@@ -1,17 +1,15 @@
 from libcpp.vector cimport vector
 
 from ..typedefs cimport hash_t
-from .kb_in_memory cimport InMemoryLookupKB
+from .kb cimport KnowledgeBase
 
 
+# Object used by the Entity Linker that summarizes one entity-alias candidate
+# combination.
 cdef class Candidate:
-    pass
-
-
-cdef class InMemoryCandidate(Candidate):
-    cdef readonly hash_t _entity_hash
-    cdef readonly hash_t _alias_hash
-    cdef vector[float] _entity_vector
-    cdef float _prior_prob
-    cdef readonly InMemoryLookupKB _kb
-    cdef float _entity_freq
+    cdef readonly KnowledgeBase kb
+    cdef hash_t entity_hash
+    cdef float entity_freq
+    cdef vector[float] entity_vector
+    cdef hash_t alias_hash
+    cdef float prior_prob

spacy/kb/candidate.pyx

@@ -1,98 +1,90 @@
 # cython: infer_types=True
 
-from .kb_in_memory cimport InMemoryLookupKB
+from typing import Iterable
 
-from ..errors import Errors
+from .kb cimport KnowledgeBase
+
+from ..tokens import Span
 
 
 cdef class Candidate:
-    """A `Candidate` object refers to a textual mention that may or may not be resolved
-    to a specific entity from a Knowledge Base. This will be used as input for the entity linking
-    algorithm which will disambiguate the various candidates to the correct one.
-    Each candidate, which represents a possible link between one textual mention and one entity in the knowledge base,
-    is assigned a certain prior probability.
+    """A `Candidate` object refers to a textual mention (`alias`) that may or
+    may not be resolved to a specific `entity` from a Knowledge Base. This
+    will be used as input for the entity linking algorithm which will
+    disambiguate the various candidates to the correct one.
+    Each candidate (alias, entity) pair is assigned a certain prior probability.
 
     DOCS: https://spacy.io/api/kb/#candidate-init
     """
 
-    def __init__(self):
-        # Make sure abstract Candidate is not instantiated.
-        if self.__class__ == Candidate:
-            raise TypeError(
-                Errors.E1046.format(cls_name=self.__class__.__name__)
-            )
-
-    @property
-    def entity_id(self) -> int:
-        """RETURNS (int): Numerical representation of entity ID (if entity ID is numerical, this is just the entity ID,
-        otherwise the hash of the entity ID string)."""
-        raise NotImplementedError
-
-    @property
-    def entity_id_(self) -> str:
-        """RETURNS (str): String representation of entity ID."""
-        raise NotImplementedError
-
-    @property
-    def entity_vector(self) -> vector[float]:
-        """RETURNS (vector[float]): Entity vector."""
-        raise NotImplementedError
-
-
-cdef class InMemoryCandidate(Candidate):
-    """Candidate for InMemoryLookupKB."""
-
     def __init__(
         self,
-        kb: InMemoryLookupKB,
-        entity_hash: int,
-        alias_hash: int,
-        entity_vector: vector[float],
-        prior_prob: float,
-        entity_freq: float
+        KnowledgeBase kb,
+        entity_hash,
+        entity_freq,
+        entity_vector,
+        alias_hash,
+        prior_prob
     ):
-        """
-        kb (InMemoryLookupKB]): InMemoryLookupKB instance.
-        entity_id (int): Entity ID as hash that can be looked up with InMemoryKB.vocab.strings.__getitem__().
-        entity_freq (int): Entity frequency in KB corpus.
-        entity_vector (List[float]): Entity embedding.
-        alias_hash (int): Alias hash.
-        prior_prob (float): Prior probability of entity for this alias. I. e. the probability that, independent of
-            the context, this alias - which matches one of this entity's aliases - resolves to one this entity.
-        """
-        super().__init__()
-
-        self._entity_hash = entity_hash
-        self._entity_vector = entity_vector
-        self._prior_prob = prior_prob
-        self._kb = kb
-        self._alias_hash = alias_hash
-        self._entity_freq = entity_freq
+        self.kb = kb
+        self.entity_hash = entity_hash
+        self.entity_freq = entity_freq
+        self.entity_vector = entity_vector
+        self.alias_hash = alias_hash
+        self.prior_prob = prior_prob
 
     @property
-    def entity_id(self) -> int:
-        return self._entity_hash
+    def entity(self) -> int:
+        """RETURNS (uint64): hash of the entity's KB ID/name"""
+        return self.entity_hash
 
     @property
-    def entity_vector(self) -> vector[float]:
-        return self._entity_vector
+    def entity_(self) -> str:
+        """RETURNS (str): ID/name of this entity in the KB"""
+        return self.kb.vocab.strings[self.entity_hash]
 
     @property
-    def prior_prob(self) -> float:
-        """RETURNS (float): Prior probability that this alias, which matches one of this entity's synonyms, resolves to
-        this entity."""
-        return self._prior_prob
+    def alias(self) -> int:
+        """RETURNS (uint64): hash of the alias"""
+        return self.alias_hash
 
     @property
-    def alias(self) -> str:
-        """RETURNS (str): Alias."""
-        return self._kb.vocab.strings[self._alias_hash]
-
-    @property
-    def entity_id_(self) -> str:
-        return self._kb.vocab.strings[self._entity_hash]
+    def alias_(self) -> str:
+        """RETURNS (str): ID of the original alias"""
+        return self.kb.vocab.strings[self.alias_hash]
 
     @property
     def entity_freq(self) -> float:
-        """RETURNS (float): Entity frequency in KB corpus."""
-        return self._entity_freq
+        return self.entity_freq
+
+    @property
+    def entity_vector(self) -> Iterable[float]:
+        return self.entity_vector
+
+    @property
+    def prior_prob(self) -> float:
+        return self.prior_prob
+
+
+def get_candidates(kb: KnowledgeBase, mention: Span) -> Iterable[Candidate]:
+    """
+    Return candidate entities for a given mention and fetching appropriate
+    entries from the index.
+    kb (KnowledgeBase): Knowledge base to query.
+    mention (Span): Entity mention for which to identify candidates.
+    RETURNS (Iterable[Candidate]): Identified candidates.
+    """
+    return kb.get_candidates(mention)
+
+
+def get_candidates_batch(
+        kb: KnowledgeBase, mentions: Iterable[Span]
+) -> Iterable[Iterable[Candidate]]:
+    """
+    Return candidate entities for the given mentions and fetching appropriate entries
+    from the index.
+    kb (KnowledgeBase): Knowledge base to query.
+    mention (Iterable[Span]): Entity mentions for which to identify candidates.
+    RETURNS (Iterable[Iterable[Candidate]]): Identified candidates.
+    """
+    return kb.get_candidates_batch(mentions)

spacy/kb/kb.pyx

@@ -1,14 +1,14 @@
 # cython: infer_types=True
 
 from pathlib import Path
-from typing import Iterable, Iterator, Tuple, Union
+from typing import Iterable, Tuple, Union
 
 from cymem.cymem cimport Pool
 
 from ..errors import Errors
-from ..tokens import SpanGroup
+from ..tokens import Span
 from ..util import SimpleFrozenList
-from .candidate cimport Candidate
+from .candidate import Candidate
 
 
 cdef class KnowledgeBase:
@@ -19,8 +19,6 @@ cdef class KnowledgeBase:
 
     DOCS: https://spacy.io/api/kb
     """
-    CandidatesForMentionT = Iterable[Candidate]
-    CandidatesForDocT = Iterable[CandidatesForMentionT]
 
     def __init__(self, vocab: Vocab, entity_vector_length: int):
         """Create a KnowledgeBase."""
@@ -34,15 +32,27 @@ cdef class KnowledgeBase:
         self.entity_vector_length = entity_vector_length
         self.mem = Pool()
 
-    def get_candidates(self, mentions: Iterator[SpanGroup]) -> Iterator[CandidatesForDocT]:
+    def get_candidates_batch(
+        self, mentions: Iterable[Span]
+    ) -> Iterable[Iterable[Candidate]]:
         """
-        Return candidate entities for the specified groups of mentions (as SpanGroup) per Doc.
-        Each candidate for a mention defines at least the entity and the entity's embedding vector. Depending on the KB
-        implementation, further properties - such as the prior probability of the specified mention text resolving to
-        that entity - might be included.
-        If no candidates are found for a given mention, an empty list is returned.
-        mentions (Iterator[SpanGroup]): Mentions for which to get candidates.
-        RETURNS (Iterator[Iterable[Iterable[Candidate]]]): Identified candidates per mention/doc/doc batch.
+        Return candidate entities for specified texts. Each candidate defines
+        the entity, the original alias, and the prior probability of that
+        alias resolving to that entity.
+        If no candidate is found for a given text, an empty list is returned.
+        mentions (Iterable[Span]): Mentions for which to get candidates.
+        RETURNS (Iterable[Iterable[Candidate]]): Identified candidates.
+        """
+        return [self.get_candidates(span) for span in mentions]
+
+    def get_candidates(self, mention: Span) -> Iterable[Candidate]:
+        """
+        Return candidate entities for specified text. Each candidate defines
+        the entity, the original alias,
+        and the prior probability of that alias resolving to that entity.
+        If the no candidate is found for a given text, an empty list is returned.
+        mention (Span): Mention for which to get candidates.
+        RETURNS (Iterable[Candidate]): Identified candidates.
         """
         raise NotImplementedError(
             Errors.E1045.format(
@@ -118,10 +128,3 @@ cdef class KnowledgeBase:
                 parent="KnowledgeBase", method="from_disk", name=self.__name__
             )
         )
-
-    @property
-    def supports_prior_probs(self) -> bool:
-        """RETURNS (bool): Whether this KB type supports looking up prior probabilities for entity mentions."""
-        raise NotImplementedError(
-            Errors.E1045.format(parent="KnowledgeBase", method="supports_prior_probs", name=self.__name__)
-        )

spacy/kb/kb_in_memory.pyx

@@ -1,5 +1,5 @@
 # cython: infer_types=True
-from typing import Any, Callable, Dict, Iterable, Iterator
+from typing import Any, Callable, Dict, Iterable
 
 import srsly
 
@@ -12,7 +12,7 @@ from preshed.maps cimport PreshMap
 import warnings
 from pathlib import Path
 
-from ..tokens import SpanGroup
+from ..tokens import Span
 
 from ..typedefs cimport hash_t
 
@@ -23,7 +23,7 @@ from ..util import SimpleFrozenList, ensure_path
 from ..vocab cimport Vocab
 from .kb cimport KnowledgeBase
 
-from .candidate import InMemoryCandidate
+from .candidate import Candidate as Candidate
 
 
 cdef class InMemoryLookupKB(KnowledgeBase):
@@ -255,11 +255,10 @@ cdef class InMemoryLookupKB(KnowledgeBase):
             alias_entry.probs = probs
             self._aliases_table[alias_index] = alias_entry
 
-    def get_candidates(self, mentions: Iterator[SpanGroup]) -> Iterator[Iterable[Iterable[InMemoryCandidate]]]:
-        for mentions_for_doc in mentions:
-            yield [self._get_alias_candidates(span.text) for span in mentions_for_doc]
+    def get_candidates(self, mention: Span) -> Iterable[Candidate]:
+        return self.get_alias_candidates(mention.text)  # type: ignore
 
-    def _get_alias_candidates(self, str alias) -> Iterable[InMemoryCandidate]:
+    def get_alias_candidates(self, str alias) -> Iterable[Candidate]:
         """
         Return candidate entities for an alias. Each candidate defines the
         entity, the original alias, and the prior probability of that alias
@@ -272,18 +271,18 @@ cdef class InMemoryLookupKB(KnowledgeBase):
         alias_index = <int64_t>self._alias_index.get(alias_hash)
         alias_entry = self._aliases_table[alias_index]
 
-        return [
-            InMemoryCandidate(
-                kb=self,
-                entity_hash=self._entries[entry_index].entity_hash,
-                alias_hash=alias_hash,
-                entity_vector=self._vectors_table[self._entries[entry_index].vector_index],
-                prior_prob=prior_prob,
-                entity_freq=self._entries[entry_index].freq
-            )
-            for (entry_index, prior_prob) in zip(alias_entry.entry_indices, alias_entry.probs)
-            if entry_index != 0
-        ]
+        return [Candidate(kb=self,
+                          entity_hash=self._entries[entry_index].entity_hash,
+                          entity_freq=self._entries[entry_index].freq,
+                          entity_vector=self._vectors_table[
+                              self._entries[entry_index].vector_index
+                          ],
+                          alias_hash=alias_hash,
+                          prior_prob=prior_prob)
+                for (entry_index, prior_prob) in zip(
+                    alias_entry.entry_indices, alias_entry.probs
+                )
+                if entry_index != 0]
 
     def get_vector(self, str entity):
         cdef hash_t entity_hash = self.vocab.strings[entity]
@@ -317,9 +316,6 @@ cdef class InMemoryLookupKB(KnowledgeBase):
 
         return 0.0
 
-    def supports_prior_probs(self) -> bool:
-        return True
-
     def to_bytes(self, **kwargs):
         """Serialize the current state to a binary string.
         """

spacy/lang/am/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "አፕል የዩኬን ጅምር ድርጅት በ 1 ቢሊዮን ዶላር ለመግዛት አስቧል።",
     "የራስ ገዝ መኪኖች የኢንሹራንስ ኃላፊነትን ወደ አምራቾች ያዛውራሉ",

spacy/lang/am/lex_attrs.py

@@ -60,7 +60,7 @@ _ordinal_words = [
     "አስራ ስምንተኛ",
     "አስራ ዘጠነኛ",
     "ሃያኛ",
-    "ሰላሳኛ" "አርባኛ",
+    "ሰላሳኛአርባኛ",
     "አምሳኛ",
     "ስድሳኛ",
     "ሰባኛ",

spacy/lang/az/examples.py

@@ -4,7 +4,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Bu bir cümlədir.",
     "Necəsən?",

spacy/lang/bg/stop_words.py

@@ -3,6 +3,7 @@ References:
     https://github.com/Alir3z4/stop-words - Original list, serves as a base.
     https://postvai.com/books/stop-dumi.pdf - Additions to the original list in order to improve it.
 """
+
 STOP_WORDS = set(
     """
 а автентичен аз ако ала

spacy/lang/bn/examples.py

@@ -5,5 +5,4 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = ["তুই খুব ভালো", "আজ আমরা ডাক্তার দেখতে যাবো", "আমি জানি না "]

spacy/lang/bo/__init__.py

@@ -0,0 +1,16 @@
+from ...language import BaseDefaults, Language
+from .lex_attrs import LEX_ATTRS
+from .stop_words import STOP_WORDS
+
+
+class TibetanDefaults(BaseDefaults):
+    lex_attr_getters = LEX_ATTRS
+    stop_words = STOP_WORDS
+
+
+class Tibetan(Language):
+    lang = "bo"
+    Defaults = TibetanDefaults
+
+
+__all__ = ["Tibetan"]

spacy/lang/bo/examples.py

@@ -0,0 +1,15 @@
+"""
+Example sentences to test spaCy and its language models.
+
+>>> from spacy.lang.bo.examples import sentences
+>>> docs = nlp.pipe(sentences)
+"""
+
+sentences = [
+    "དོན་དུ་རྒྱ་མཚོ་བླ་མ་ཞེས་བྱ་ཞིང༌།",
+    "ཏཱ་ལའི་ཞེས་པ་ནི་སོག་སྐད་ཡིན་པ་དེ་བོད་སྐད་དུ་རྒྱ་མཚོའི་དོན་དུ་འཇུག",
+    "སོག་པོ་ཨལ་ཐན་རྒྱལ་པོས་རྒྱལ་དབང་བསོད་ནམས་རྒྱ་མཚོར་ཆེ་བསྟོད་ཀྱི་མཚན་གསོལ་བ་ཞིག་ཡིན་ཞིང༌།",
+    "རྗེས་སུ་རྒྱལ་བ་དགེ་འདུན་གྲུབ་དང༌། དགེ་འདུན་རྒྱ་མཚོ་སོ་སོར་ཡང་ཏཱ་ལའི་བླ་མའི་སྐུ་ཕྲེང་དང་པོ་དང༌།",
+    "གཉིས་པའི་མཚན་དེ་གསོལ་ཞིང༌།༸རྒྱལ་དབང་སྐུ་ཕྲེང་ལྔ་པས་དགའ་ལྡན་ཕོ་བྲང་གི་སྲིད་དབང་བཙུགས་པ་ནས་ཏཱ་ལའི་བླ་མ་ནི་བོད་ཀྱི་ཆོས་སྲིད་གཉིས་ཀྱི་དབུ་ཁྲིད་དུ་གྱུར་ཞིང་།",
+    "ད་ལྟའི་བར་ཏཱ་ལའི་བླ་མ་སྐུ་ཕྲེང་བཅུ་བཞི་བྱོན་ཡོད།",
+]

spacy/lang/bo/lex_attrs.py

@@ -0,0 +1,65 @@
+from ...attrs import LIKE_NUM
+
+# reference 1: https://en.wikipedia.org/wiki/Tibetan_numerals
+
+_num_words = [
+    "ཀླད་ཀོར་",
+    "གཅིག་",
+    "གཉིས་",
+    "གསུམ་",
+    "བཞི་",
+    "ལྔ་",
+    "དྲུག་",
+    "བདུན་",
+    "བརྒྱད་",
+    "དགུ་",
+    "བཅུ་",
+    "བཅུ་གཅིག་",
+    "བཅུ་གཉིས་",
+    "བཅུ་གསུམ་",
+    "བཅུ་བཞི་",
+    "བཅུ་ལྔ་",
+    "བཅུ་དྲུག་",
+    "བཅུ་བདུན་",
+    "བཅུ་པརྒྱད",
+    "བཅུ་དགུ་",
+    "ཉི་ཤུ་",
+    "སུམ་ཅུ",
+    "བཞི་བཅུ",
+    "ལྔ་བཅུ",
+    "དྲུག་ཅུ",
+    "བདུན་ཅུ",
+    "བརྒྱད་ཅུ",
+    "དགུ་བཅུ",
+    "བརྒྱ་",
+    "སྟོང་",
+    "ཁྲི་",
+    "ས་ཡ་",
+    "	བྱེ་བ་",
+    "དུང་ཕྱུར་",
+    "ཐེར་འབུམ་",
+    "ཐེར་འབུམ་ཆེན་པོ་",
+    "ཁྲག་ཁྲིག་",
+    "ཁྲག་ཁྲིག་ཆེན་པོ་",
+]
+
+
+def like_num(text):
+    """
+    Check if text resembles a number
+    """
+    if text.startswith(("+", "-", "±", "~")):
+        text = text[1:]
+    text = text.replace(",", "").replace(".", "")
+    if text.isdigit():
+        return True
+    if text.count("/") == 1:
+        num, denom = text.split("/")
+        if num.isdigit() and denom.isdigit():
+            return True
+    if text in _num_words:
+        return True
+    return False
+
+
+LEX_ATTRS = {LIKE_NUM: like_num}

spacy/lang/bo/stop_words.py

@@ -0,0 +1,198 @@
+# Source: https://zenodo.org/records/10148636
+
+STOP_WORDS = set(
+    """
+འི་
+།
+དུ་
+གིས་
+སོགས་
+ཏེ
+གི་
+རྣམས་
+ནི
+ཀུན་
+ཡི་
+འདི
+ཀྱི་
+སྙེད་
+པས་
+གཞན་
+ཀྱིས་
+ཡི
+ལ
+ནི་
+དང་
+སོགས
+ཅིང་
+ར
+དུ
+མི་
+སུ་
+བཅས་
+ཡོངས་
+ལས
+ཙམ་
+གྱིས་
+དེ་
+ཡང་
+མཐའ་དག་
+ཏུ་
+ཉིད་
+ས
+ཏེ་
+གྱི་
+སྤྱི
+དེ
+ཀ་
+ཡིན་
+ཞིང་
+འདི་
+རུང་
+རང་
+ཞིག་
+སྟེ
+སྟེ་
+ན་རེ
+ངམ
+ཤིང་
+དག་
+ཏོ
+རེ་
+འང་
+ཀྱང་
+ལགས་པ
+ཚུ
+དོ
+ཡིན་པ
+རེ
+ན་རེ་
+ཨེ་
+ཚང་མ
+ཐམས་ཅད་
+དམ་
+འོ་
+ཅིག་
+གྱིན་
+ཡིན
+ན
+ཁོ་ན་
+འམ་
+ཀྱིན་
+ལོ
+ཀྱིས
+བས་
+ལགས་
+ཤིག
+གིས
+ཀི་
+སྣ་ཚོགས་
+རྣམས
+སྙེད་པ
+ཡིས་
+གྱི
+གི
+བམ་
+ཤིག་
+རེ་རེ་
+ནམ
+མིན་
+ནམ་
+ངམ་
+རུ་
+འགའ་
+ཀུན
+ཤས་
+ཏུ
+ཡིས
+གིན་
+གམ་
+འོ
+ཡིན་པ་
+མིན
+ལགས
+གྱིས
+ཅང་
+འགའ
+སམ་
+ཞིག
+འང
+ལས་ཆེ་
+འཕྲལ་
+བར་
+རུ
+དང
+ཡ
+འག
+སམ
+ཀ
+ཅུང་ཟད་
+ཅིག
+ཉིད
+དུ་མ
+མ
+ཡིན་བ
+འམ
+མམ
+དམ
+དག
+ཁོ་ན
+ཀྱི
+ལམ
+ཕྱི་
+ནང་
+ཙམ
+ནོ་
+སོ་
+རམ་
+བོ་
+ཨང་
+ཕྱི
+ཏོ་
+ཚོ
+ལ་ལ་
+ཚོ་
+ཅིང
+མ་གི་
+གེ
+གོ
+ཡིན་ལུགས་
+རོ་
+བོ
+ལགས་པ་
+པས
+རབ་
+འི
+རམ
+བས
+གཞན
+སྙེད་པ་
+འབའ་
+མཾ་
+པོ
+ག་
+ག
+གམ
+སྤྱི་
+བམ
+མོ་
+ཙམ་པ་
+ཤ་སྟག་
+མམ་
+རེ་རེ
+སྙེད
+ཏམ་
+ངོ
+གྲང་
+ཏ་རེ
+ཏམ
+ཁ་
+ངེ་
+ཅོག་
+རིལ་
+ཉུང་ཤས་
+གིང་
+ཚ་
+ཀྱང
+""".split()
+)

spacy/lang/ca/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Apple està buscant comprar una startup del Regne Unit per mil milions de dòlars",
     "Els cotxes autònoms deleguen la responsabilitat de l'assegurança als seus fabricants",

spacy/lang/char_classes.py

@@ -277,10 +277,10 @@ _currency = (
 # These expressions contain various unicode variations, including characters
 # used in Chinese (see #1333, #1340, #1351) – unless there are cross-language
 # conflicts, spaCy's base tokenizer should handle all of those by default
-_punct = (
-    r"… …… , : ; \! \? ¿ ؟ ¡ \( \) \[ \] \{ \} < > _ # \* & 。 ？ ！ ， 、 ； ： ～ · । ، ۔ ؛ ٪"
+_punct = r"… …… , : ; \! \? ¿ ؟ ¡ \( \) \[ \] \{ \} < > _ # \* & 。 ？ ！ ， 、 ； ： ～ · । ، ۔ ؛ ٪"
+_quotes = (
+    r'\' " ” “ ` ‘ ´ ’ ‚ , „ » « 「 」 『 』 （ ） 〔 〕 【 】 《 》 〈 〉 〈 〉  ⟦ ⟧'
 )
-_quotes = r'\' " ” “ ` ‘ ´ ’ ‚ , „ » « 「 」 『 』 （ ） 〔 〕 【 】 《 》 〈 〉 〈 〉  ⟦ ⟧'
 _hyphens = "- – — -- --- —— ~"
 
 # Various symbols like dingbats, but also emoji

spacy/lang/cs/examples.py

@@ -4,7 +4,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Máma mele maso.",
     "Příliš žluťoučký kůň úpěl ďábelské ódy.",

spacy/lang/da/tokenizer_exceptions.py

@@ -2,6 +2,7 @@
 Tokenizer Exceptions.
 Source: https://forkortelse.dk/ and various others.
 """
+
 from ...symbols import NORM, ORTH
 from ...util import update_exc
 from ..tokenizer_exceptions import BASE_EXCEPTIONS

spacy/lang/de/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Die ganze Stadt ist ein Startup: Shenzhen ist das Silicon Valley für Hardware-Firmen",
     "Wie deutsche Startups die Technologie vorantreiben wollen: Künstliche Intelligenz",

spacy/lang/dsb/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Z tym stwori so wuměnjenje a zakład za dalše wobdźěłanje přez analyzu tekstoweje struktury a semantisku anotaciju a z tym tež za tu předstajenu digitalnu online-wersiju.",
     "Mi so tu jara derje spodoba.",

spacy/lang/el/tokenizer_exceptions.py

@@ -128,7 +128,6 @@ _other_exc = {
 _exc.update(_other_exc)
 
 for h in range(1, 12 + 1):
-
     for period in ["π.μ.", "πμ"]:
         _exc[f"{h}{period}"] = [
             {ORTH: f"{h}"},

spacy/lang/en/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Apple is looking at buying U.K. startup for $1 billion",
     "Autonomous cars shift insurance liability toward manufacturers",

spacy/lang/es/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Apple está buscando comprar una startup del Reino Unido por mil millones de dólares.",
     "Los coches autónomos delegan la responsabilidad del seguro en sus fabricantes.",

spacy/lang/es/lemmatizer.py

@@ -63,16 +63,13 @@ class SpanishLemmatizer(Lemmatizer):
         self.cache[cache_key] = lemmas
         return lemmas
 
-    def select_rule(self, pos: str, features: List[str]) -> str:
+    def select_rule(self, pos: str, features: List[str]) -> Optional[str]:
         groups = self.lookups.get_table("lemma_rules_groups")
         if pos in groups:
             for group in groups[pos]:
                 if set(group[1]).issubset(features):
                     return group[0]
-        # In v3, returning None here apparently took advantage of a bug in the string store
-        # that didn't raise an error on None as a value to decode. We emulate the previous
-        # behaviour by returning "" here, which should not match any lookups as before.
-        return ""
+        return None
 
     def lemmatize_adj(
         self, word: str, features: List[str], rule: str, index: List[str]
@@ -418,7 +415,10 @@ class SpanishLemmatizer(Lemmatizer):
         else:
             rule = self.select_rule("verb", features)
             verb_lemma = self.lemmatize_verb(
-                verb, features - {"PronType=Prs"}, rule, index  # type: ignore[operator]
+                verb,
+                features - {"PronType=Prs"},  # type: ignore[operator]
+                rule,
+                index,  # type: ignore[operator]
             )[0]
         pron_lemmas = []
         for pron in prons:

spacy/lang/fa/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "این یک جمله نمونه می باشد.",
     "قرار ما، امروز ساعت ۲:۳۰ بعدازظهر هست!",

spacy/lang/fa/generate_verbs_exc.py

@@ -611,8 +611,8 @@ narrative_ends = ["ه‌ام", "ه‌ای", "ه", "ه‌ایم", "ه‌اید",
 present_ends = ["م", "ی", "د", "یم", "ید", "ند"]
 
 # special case of '#هست':
-VERBS_EXC.update({conj: "هست" for conj in ["هست" + end for end in simple_ends]})
-VERBS_EXC.update({conj: "هست" for conj in ["نیست" + end for end in simple_ends]})
+VERBS_EXC.update(dict.fromkeys(["هست" + end for end in simple_ends], "هست"))
+VERBS_EXC.update(dict.fromkeys(["نیست" + end for end in simple_ends], "هست"))
 
 for verb_root in verb_roots:
     conjugations = []
@@ -648,4 +648,4 @@ for verb_root in verb_roots:
             )
         )
 
-    VERBS_EXC.update({conj: (past,) if past else present for conj in conjugations})
+    VERBS_EXC.update(dict.fromkeys(conjugations, (past,) if past else present))

spacy/lang/fi/tokenizer_exceptions.py

@@ -100,9 +100,9 @@ conj_contraction_negations = [
     ("eivat", "eivät"),
     ("eivät", "eivät"),
 ]
-for (base_lower, base_norm) in conj_contraction_bases:
+for base_lower, base_norm in conj_contraction_bases:
     for base in [base_lower, base_lower.title()]:
-        for (suffix, suffix_norm) in conj_contraction_negations:
+        for suffix, suffix_norm in conj_contraction_negations:
             _exc[base + suffix] = [
                 {ORTH: base, NORM: base_norm},
                 {ORTH: suffix, NORM: suffix_norm},

spacy/lang/fr/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Apple cherche à acheter une start-up anglaise pour 1 milliard de dollars",
     "Les voitures autonomes déplacent la responsabilité de l'assurance vers les constructeurs",

spacy/lang/ga/lemmatizer.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Tuple
+from typing import List, Tuple
 
 from ...pipeline import Lemmatizer
 from ...tokens import Token

spacy/lang/gd/__init__.py

@@ -0,0 +1,18 @@
+from typing import Optional
+
+from ...language import BaseDefaults, Language
+from .stop_words import STOP_WORDS
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
+
+
+class ScottishDefaults(BaseDefaults):
+    tokenizer_exceptions = TOKENIZER_EXCEPTIONS
+    stop_words = STOP_WORDS
+
+
+class Scottish(Language):
+    lang = "gd"
+    Defaults = ScottishDefaults
+
+
+__all__ = ["Scottish"]

spacy/lang/gd/stop_words.py

@@ -0,0 +1,386 @@
+STOP_WORDS = set(
+    """
+'ad
+'ar
+'d # iad
+'g # ag
+'ga
+'gam
+'gan
+'gar
+'gur
+'m # am
+'n # an
+'n seo
+'na
+'nad
+'nam
+'nan
+'nar
+'nuair
+'nur
+'s
+'sa
+'san
+'sann
+'se
+'sna
+a
+a'
+a'd # agad
+a'm # agam
+a-chèile
+a-seo
+a-sin
+a-siud
+a chionn
+a chionn 's
+a chèile
+a chéile
+a dh'
+a h-uile
+a seo
+ac' # aca
+aca
+aca-san
+acasan
+ach
+ag
+agad
+agad-sa
+agads'
+agadsa
+agaibh
+agaibhse
+againn
+againne
+agam
+agam-sa
+agams'
+agamsa
+agus
+aice
+aice-se
+aicese
+aig
+aig' # aige
+aige
+aige-san
+aigesan
+air
+air-san
+air neo
+airsan
+am
+an
+an seo
+an sin
+an siud
+an uair
+ann
+ann a
+ann a'
+ann a shin
+ann am
+ann an
+annad
+annam
+annam-s'
+annamsa
+anns
+anns an
+annta
+aon
+ar
+as
+asad
+asda
+asta
+b'
+bho
+bhon
+bhuaidhe # bhuaithe
+bhuainn
+bhuaipe
+bhuaithe
+bhuapa
+bhur
+brì
+bu
+c'à
+car son
+carson
+cha
+chan
+chionn
+choir
+chon
+chun
+chèile
+chéile
+chòir
+cia mheud
+ciamar
+co-dhiubh
+cuide
+cuin
+cuin'
+cuine
+cà
+cà'
+càil
+càit
+càit'
+càite
+cò
+cò mheud
+có
+d'
+da
+de
+dh'
+dha
+dhaibh
+dhaibh-san
+dhaibhsan
+dhan
+dhasan
+dhe
+dhen
+dheth
+dhi
+dhiom
+dhiot
+dhith
+dhiubh
+dhomh
+dhomh-s'
+dhomhsa
+dhu'sa # dhut-sa
+dhuibh
+dhuibhse
+dhuinn
+dhuinne
+dhuit
+dhut
+dhutsa
+dhut-sa
+dhà
+dhà-san
+dhàsan
+dhòmhsa
+diubh
+do
+docha
+don
+dà
+dè
+dè mar
+dé
+dé mar
+dòch'
+dòcha
+e
+eadar
+eatarra
+eatorra
+eile
+esan
+fa
+far
+feud
+fhad
+fheudar
+fhearr
+fhein
+fheudar
+fheàrr
+fhèin
+fhéin
+fhìn
+fo
+fodha
+fodhainn
+foipe
+fon
+fèin
+ga
+gach
+gam
+gan
+ge brith
+ged
+gu
+gu dè
+gu ruige
+gun
+gur
+gus
+i
+iad
+iadsan
+innte
+is
+ise
+le
+leam
+leam-sa
+leamsa
+leat
+leat-sa
+leatha
+leatsa
+leibh
+leis
+leis-san
+leoth'
+leotha
+leotha-san
+linn
+m'
+m'a
+ma
+mac
+man
+mar
+mas
+mathaid
+mi
+mis'
+mise
+mo
+mu
+mu 'n
+mun
+mur
+mura
+mus
+na
+na b'
+na bu
+na iad
+nach
+nad
+nam
+nan
+nar
+nas
+neo
+no
+nuair
+o
+o'n
+oir
+oirbh
+oirbh-se
+oirnn
+oirnne
+oirre
+on
+orm
+orm-sa
+ormsa
+orra
+orra-san
+orrasan
+ort
+os
+r'
+ri
+ribh
+rinn
+ris
+rithe
+rithe-se
+rium
+rium-sa
+riums'
+riumsa
+riut
+riuth'
+riutha
+riuthasan
+ro
+ro'n
+roimh
+roimhe
+romhainn
+romham
+romhpa
+ron
+ruibh
+ruinn
+ruinne
+sa
+san
+sann
+se
+seach
+seo
+seothach
+shin
+sibh
+sibh-se
+sibhse
+sin
+sineach
+sinn
+sinne
+siod
+siodach
+siud
+siudach
+sna # ann an
+sè
+t'
+tarsaing
+tarsainn
+tarsuinn
+thar
+thoigh
+thro
+thu
+thuc'
+thuca
+thugad
+thugaibh
+thugainn
+thugam
+thugamsa
+thuice
+thuige
+thus'
+thusa
+timcheall
+toigh
+toil
+tro
+tro' # troimh
+troimh
+troimhe
+tron
+tu
+tusa
+uair
+ud
+ugaibh
+ugam-s'
+ugam-sa
+uice
+uige
+uige-san
+umad
+unnta # ann an
+ur
+urrainn
+à
+às
+àsan
+á
+ás
+è
+ì
+ò
+ó
+""".split("\n")
+)

spacy/lang/grc/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "ἐρᾷ μὲν ἁγνὸς οὐρανὸς τρῶσαι χθόνα, ἔρως δὲ γαῖαν λαμβάνει γάμου τυχεῖν·",
     "εὐδαίμων Χαρίτων καὶ Μελάνιππος ἔφυ, θείας ἁγητῆρες ἐφαμερίοις φιλότατος.",

spacy/lang/gu/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "લોકશાહી એ સરકારનું એક એવું તંત્ર છે જ્યાં નાગરિકો મત દ્વારા સત્તાનો ઉપયોગ કરે છે.",
     "તે ગુજરાત રાજ્યના ધરમપુર શહેરમાં આવેલું હતું",

spacy/lang/he/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "סין מקימה קרן של 440 מיליון דולר להשקעה בהייטק בישראל",
     'רה"מ הודיע כי יחרים טקס בחסותו',

spacy/lang/hi/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "एप्पल 1 अरब डॉलर के लिए यू.के. स्टार्टअप खरीदने पर विचार कर रहा है।",
     "स्वायत्त कारें निर्माताओं की ओर बीमा दायित्व रखतीं हैं।",

spacy/lang/hr/lemma_lookup_license.txt

@@ -1,5 +1,5 @@
 The list of Croatian lemmas was extracted from the reldi-tagger repository (https://github.com/clarinsi/reldi-tagger).
-Reldi-tagger is licesned under the Apache 2.0 licence.
+Reldi-tagger is licensed under the Apache 2.0 licence.
 
 @InProceedings{ljubesic16-new,
   author = {Nikola Ljubešić and Filip Klubička and Željko Agić and Ivo-Pavao Jazbec},
@@ -12,4 +12,4 @@ Reldi-tagger is licesned under the Apache 2.0 licence.
   publisher = {European Language Resources Association (ELRA)},
   address = {Paris, France},
   isbn = {978-2-9517408-9-1}
- }
+ }

spacy/lang/hsb/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "To běšo wjelgin raźone a jo se wót luźi derje pśiwzeło. Tak som dožywiła wjelgin",
     "Jogo pśewóźowarce stej groniłej, až how w serbskich stronach njama Santa Claus nic pytaś.",

spacy/lang/ht/__init__.py

@@ -0,0 +1,55 @@
+from typing import Callable, Optional
+
+from thinc.api import Model
+
+from ...language import BaseDefaults, Language
+from .lemmatizer import HaitianCreoleLemmatizer
+from .lex_attrs import LEX_ATTRS
+from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES
+from .stop_words import STOP_WORDS
+from .syntax_iterators import SYNTAX_ITERATORS
+from .tag_map import TAG_MAP
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
+
+
+class HaitianCreoleDefaults(BaseDefaults):
+    tokenizer_exceptions = TOKENIZER_EXCEPTIONS
+    prefixes = TOKENIZER_PREFIXES
+    infixes = TOKENIZER_INFIXES
+    suffixes = TOKENIZER_SUFFIXES
+    lex_attr_getters = LEX_ATTRS
+    syntax_iterators = SYNTAX_ITERATORS
+    stop_words = STOP_WORDS
+    tag_map = TAG_MAP
+
+
+class HaitianCreole(Language):
+    lang = "ht"
+    Defaults = HaitianCreoleDefaults
+
+
+@HaitianCreole.factory(
+    "lemmatizer",
+    assigns=["token.lemma"],
+    default_config={
+        "model": None,
+        "mode": "rule",
+        "overwrite": False,
+        "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"},
+    },
+    default_score_weights={"lemma_acc": 1.0},
+)
+def make_lemmatizer(
+    nlp: Language,
+    model: Optional[Model],
+    name: str,
+    mode: str,
+    overwrite: bool,
+    scorer: Optional[Callable],
+):
+    return HaitianCreoleLemmatizer(
+        nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer
+    )
+
+
+__all__ = ["HaitianCreole"]

spacy/lang/ht/examples.py

@@ -0,0 +1,17 @@
+"""
+Example sentences to test spaCy and its language models.
+
+>>> from spacy.lang.ht.examples import sentences
+>>> docs = nlp.pipe(sentences)
+"""
+
+sentences = [
+    "Apple ap panse achte yon demaraj nan Wayòm Ini pou $1 milya dola",
+    "Machin otonòm fè responsablite asirans lan ale sou men fabrikan yo",
+    "San Francisco ap konsidere entèdi robo ki livre sou twotwa yo",
+    "Lond se yon gwo vil nan Wayòm Ini",
+    "Kote ou ye?",
+    "Kilès ki prezidan Lafrans?",
+    "Ki kapital Etazini?",
+    "Kile Barack Obama te fèt?",
+]

spacy/lang/ht/lemmatizer.py

@@ -0,0 +1,50 @@
+from typing import List, Tuple
+
+from ...pipeline import Lemmatizer
+from ...tokens import Token
+
+
+class HaitianCreoleLemmatizer(Lemmatizer):
+    """
+    Minimal Haitian Creole lemmatizer.
+    Returns a word's base form based on rules and lookup,
+    or defaults to the original form.
+    """
+
+    def is_base_form(self, token: Token) -> bool:
+        morph = token.morph.to_dict()
+        upos = token.pos_.lower()
+
+        # Consider unmarked forms to be base
+        if upos in {"noun", "verb", "adj", "adv"}:
+            if not morph:
+                return True
+            if upos == "noun" and morph.get("Number") == "Sing":
+                return True
+            if upos == "verb" and morph.get("VerbForm") == "Inf":
+                return True
+            if upos == "adj" and morph.get("Degree") == "Pos":
+                return True
+        return False
+
+    def rule_lemmatize(self, token: Token) -> List[str]:
+        string = token.text.lower()
+        pos = token.pos_.lower()
+        cache_key = (token.orth, token.pos)
+        if cache_key in self.cache:
+            return self.cache[cache_key]
+
+        forms = []
+
+        # fallback rule: just return lowercased form
+        forms.append(string)
+
+        self.cache[cache_key] = forms
+        return forms
+
+    @classmethod
+    def get_lookups_config(cls, mode: str) -> Tuple[List[str], List[str]]:
+        if mode == "rule":
+            required = ["lemma_lookup", "lemma_rules", "lemma_exc", "lemma_index"]
+            return (required, [])
+        return super().get_lookups_config(mode)

spacy/lang/ht/lex_attrs.py

@@ -0,0 +1,81 @@
+from ...attrs import LIKE_NUM, NORM
+
+# Cardinal numbers in Creole
+_num_words = set(
+    """
+zewo youn en de twa kat senk sis sèt uit nèf dis
+onz douz trèz katoz kenz sèz disèt dizwit diznèf
+vent trant karant sinkant swasant swasann-dis
+san mil milyon milya
+""".split()
+)
+
+# Ordinal numbers in Creole (some are French-influenced, some simplified)
+_ordinal_words = set(
+    """
+premye dezyèm twazyèm katryèm senkyèm sizyèm sètvyèm uitvyèm nèvyèm dizyèm
+onzèm douzyèm trèzyèm katozyèm kenzèm sèzyèm disetyèm dizwityèm diznèvyèm
+ventyèm trantyèm karantyèm sinkantyèm swasantyèm
+swasann-disyèm santyèm milyèm milyonnyèm milyadyèm
+""".split()
+)
+
+NORM_MAP = {
+    "'m": "mwen",
+    "'w": "ou",
+    "'l": "li",
+    "'n": "nou",
+    "'y": "yo",
+    "’m": "mwen",
+    "’w": "ou",
+    "’l": "li",
+    "’n": "nou",
+    "’y": "yo",
+    "m": "mwen",
+    "n": "nou",
+    "l": "li",
+    "y": "yo",
+    "w": "ou",
+    "t": "te",
+    "k": "ki",
+    "p": "pa",
+    "M": "Mwen",
+    "N": "Nou",
+    "L": "Li",
+    "Y": "Yo",
+    "W": "Ou",
+    "T": "Te",
+    "K": "Ki",
+    "P": "Pa",
+}
+
+
+def like_num(text):
+    text = text.strip().lower()
+    if text.startswith(("+", "-", "±", "~")):
+        text = text[1:]
+    text = text.replace(",", "").replace(".", "")
+    if text.isdigit():
+        return True
+    if text.count("/") == 1:
+        num, denom = text.split("/")
+        if num.isdigit() and denom.isdigit():
+            return True
+    if text in _num_words:
+        return True
+    if text in _ordinal_words:
+        return True
+    # Handle things like "3yèm", "10yèm", "25yèm", etc.
+    if text.endswith("yèm") and text[:-3].isdigit():
+        return True
+    return False
+
+
+def norm_custom(text):
+    return NORM_MAP.get(text, text.lower())
+
+
+LEX_ATTRS = {
+    LIKE_NUM: like_num,
+    NORM: norm_custom,
+}

spacy/lang/ht/punctuation.py

@@ -0,0 +1,58 @@
+from ..char_classes import (
+    ALPHA,
+    ALPHA_LOWER,
+    ALPHA_UPPER,
+    CONCAT_QUOTES,
+    HYPHENS,
+    LIST_ELLIPSES,
+    LIST_ICONS,
+    LIST_PUNCT,
+    LIST_QUOTES,
+    merge_chars,
+)
+
+ELISION = "'’".replace(" ", "")
+
+_prefixes_elision = "m n l y t k w"
+_prefixes_elision += " " + _prefixes_elision.upper()
+
+TOKENIZER_PREFIXES = (
+    LIST_PUNCT
+    + LIST_QUOTES
+    + [
+        r"(?:({pe})[{el}])(?=[{a}])".format(
+            a=ALPHA, el=ELISION, pe=merge_chars(_prefixes_elision)
+        )
+    ]
+)
+
+TOKENIZER_SUFFIXES = (
+    LIST_PUNCT
+    + LIST_QUOTES
+    + LIST_ELLIPSES
+    + [
+        r"(?<=[0-9])%",  # numbers like 10%
+        r"(?<=[0-9])(?:{h})".format(h=HYPHENS),  # hyphens after numbers
+        r"(?<=[{a}])['’]".format(a=ALPHA),  # apostrophes after letters
+        r"(?<=[{a}])['’][mwlnytk](?=\s|$)".format(a=ALPHA),  # contractions
+        r"(?<=[{a}0-9])\)",  # right parenthesis after letter/number
+        r"(?<=[{a}])\.(?=\s|$)".format(
+            a=ALPHA
+        ),  # period after letter if space or end of string
+        r"(?<=\))[\.\?!]",  # punctuation immediately after right parenthesis
+    ]
+)
+
+TOKENIZER_INFIXES = (
+    LIST_ELLIPSES
+    + LIST_ICONS
+    + [
+        r"(?<=[0-9])[+\-\*^](?=[0-9-])",
+        r"(?<=[{al}{q}])\.(?=[{au}{q}])".format(
+            al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES
+        ),
+        r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA),
+        r"(?<=[{a}0-9])(?:{h})(?=[{a}])".format(a=ALPHA, h=HYPHENS),
+        r"(?<=[{a}][{el}])(?=[{a}])".format(a=ALPHA, el=ELISION),
+    ]
+)

spacy/lang/ht/stop_words.py

@@ -0,0 +1,49 @@
+STOP_WORDS = set(
+    """
+a ak an ankò ant apre ap atò avan avanlè
+byen bò byenke
+
+chak
+
+de depi deja deja
+
+e en epi èske
+
+fò fòk
+
+gen genyen
+
+ki kisa kilès kote koukou konsa konbyen konn konnen kounye kouman
+
+la l laa le lè li lye lò
+
+m m' mwen
+
+nan nap nou n'
+
+ou oumenm
+
+pa paske pami pandan pito pou pral preske pwiske
+
+se selman si sou sòt
+
+ta tap tankou te toujou tou tan tout toutotan twòp tèl
+
+w w' wi wè
+
+y y' yo yon yonn
+
+non o oh eh
+
+sa san si swa si
+
+men mèsi oswa osinon
+
+""".split()
+)
+
+# Add common contractions, with and without apostrophe variants
+contractions = ["m'", "n'", "w'", "y'", "l'", "t'", "k'"]
+for apostrophe in ["'", "’", "‘"]:
+    for word in contractions:
+        STOP_WORDS.add(word.replace("'", apostrophe))

spacy/lang/ht/syntax_iterators.py

@@ -0,0 +1,74 @@
+from typing import Iterator, Tuple, Union
+
+from ...errors import Errors
+from ...symbols import NOUN, PRON, PROPN
+from ...tokens import Doc, Span
+
+
+def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]:
+    """
+    Detect base noun phrases from a dependency parse for Haitian Creole.
+    Works on both Doc and Span objects.
+    """
+
+    # Core nominal dependencies common in Haitian Creole
+    labels = [
+        "nsubj",
+        "obj",
+        "obl",
+        "nmod",
+        "appos",
+        "ROOT",
+    ]
+
+    # Modifiers to optionally include in chunk (to the right)
+    post_modifiers = ["compound", "flat", "flat:name", "fixed"]
+
+    doc = doclike.doc
+    if not doc.has_annotation("DEP"):
+        raise ValueError(Errors.E029)
+
+    np_deps = {doc.vocab.strings.add(label) for label in labels}
+    np_mods = {doc.vocab.strings.add(mod) for mod in post_modifiers}
+    conj_label = doc.vocab.strings.add("conj")
+    np_label = doc.vocab.strings.add("NP")
+    adp_pos = doc.vocab.strings.add("ADP")
+    cc_pos = doc.vocab.strings.add("CCONJ")
+
+    prev_end = -1
+    for i, word in enumerate(doclike):
+        if word.pos not in (NOUN, PROPN, PRON):
+            continue
+        if word.left_edge.i <= prev_end:
+            continue
+
+        if word.dep in np_deps:
+            right_end = word
+            # expand to include known modifiers to the right
+            for child in word.rights:
+                if child.dep in np_mods:
+                    right_end = child.right_edge
+                elif child.pos == NOUN:
+                    right_end = child.right_edge
+
+            left_index = word.left_edge.i
+            # Skip prepositions at the start
+            if word.left_edge.pos == adp_pos:
+                left_index += 1
+
+            prev_end = right_end.i
+            yield left_index, right_end.i + 1, np_label
+
+        elif word.dep == conj_label:
+            head = word.head
+            while head.dep == conj_label and head.head.i < head.i:
+                head = head.head
+            if head.dep in np_deps:
+                left_index = word.left_edge.i
+                if word.left_edge.pos == cc_pos:
+                    left_index += 1
+                prev_end = word.i
+                yield left_index, word.i + 1, np_label
+
+
+SYNTAX_ITERATORS = {"noun_chunks": noun_chunks}