Install click with previous spaCy in upgrade test

spacy <=3.8.14 imports click without declaring it (#13971), and modern
typer no longer depends on click, so the pre-upgrade install fails on
import without it.

.github/spacy_universe_alert.py

@@ -10,7 +10,7 @@ SLACK_TOKEN = os.environ.get("SLACK_BOT_TOKEN", "ENV VAR not available!")
 DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
 
 client = WebClient(SLACK_TOKEN)
-github_context = json.loads(sys.argv[1])
+github_context = json.loads(sys.stdin.read())
 
 event = github_context['event']
 pr_title = event['pull_request']["title"]

.github/workflows/cibuildwheel.yml

@@ -0,0 +1,114 @@
+name: Build
+
+on:
+  push:
+    tags:
+      # ytf did they invent their own syntax that's almost regex?
+      # ** matches 'zero or more of any character'
+      - 'release-v[0-9]+.[0-9]+.[0-9]+**'
+      - 'prerelease-v[0-9]+.[0-9]+.[0-9]+**'
+
+permissions: {}
+
+jobs:
+  build_wheels:
+    uses: explosion/gha-cibuildwheel/.github/workflows/cibuildwheel.yml@2c98f757f13d112cf73fcf4b627249f1fffb5aae  # main
+    permissions:
+      contents: write
+      actions: read
+    with:
+      wheel-name-pattern: "spacy-*.whl"
+      pure-python: false
+    secrets:
+      gh-token: ${{ secrets.GITHUB_TOKEN }}
+
+  smoke_test:
+    name: Smoke test
+    # No checkout here: the jobs below must import the installed wheel, and a
+    # checked-out source tree in the working directory would shadow it.
+    needs: build_wheels
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.12"
+
+      # The release created by build_wheels is a draft, which can't be looked
+      # up by tag, so pull the wheel from this run's artifacts instead
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8
+        with:
+          pattern: "cibw-wheels-ubuntu-latest-*"
+          merge-multiple: true
+          path: "dist"
+
+      - name: Install from wheel
+        run: |
+          WHEEL=$(ls dist/spacy-*cp312*manylinux*x86_64*.whl | head -1)
+          pip install "$WHEEL"
+
+      - name: Test import
+        run: python -c "import spacy; print('spacy==' + spacy.__version__)"
+
+      - name: Download and load model
+        run: |
+          python -m spacy download en_core_web_sm
+          python -c "
+          import spacy
+          nlp = spacy.load('en_core_web_sm')
+          doc = nlp('Apple is looking at buying U.K. startup for \$1 billion')
+          assert len(doc.ents) > 0, 'No entities found'
+          print('Model load OK:', nlp.meta['name'], '@', nlp.meta['version'])
+          print('Entities:', [(ent.text, ent.label_) for ent in doc.ents])
+          "
+
+  upgrade_test:
+    name: Upgrade test
+    # No checkout here: the jobs below must import the installed wheel, and a
+    # checked-out source tree in the working directory would shadow it.
+    needs: build_wheels
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.12"
+
+      - name: Install previous spaCy version
+        run: |
+          # strip the release-v/prerelease-v prefix to get a version specifier;
+          # click is needed because spacy <=3.8.14 imports it without
+          # declaring it (#13971) and modern typer no longer pulls it in
+          pip install "spacy>=3.8.0,<${GITHUB_REF_NAME##*-v}" click || pip install "spacy<4" click
+          python -m spacy download en_core_web_sm
+          python -c "
+          import spacy
+          nlp = spacy.load('en_core_web_sm')
+          print('Pre-upgrade:', spacy.__version__, nlp.meta['name'], '@', nlp.meta['version'])
+          "
+
+      # The release created by build_wheels is a draft, which can't be looked
+      # up by tag, so pull the wheel from this run's artifacts instead
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8
+        with:
+          pattern: "cibw-wheels-ubuntu-latest-*"
+          merge-multiple: true
+          path: "dist"
+
+      - name: Upgrade to new version
+        run: |
+          WHEEL=$(ls dist/spacy-*cp312*manylinux*x86_64*.whl | head -1)
+          pip install "$WHEEL"
+
+      - name: Test model still loads after upgrade
+        run: |
+          python -c "
+          import spacy
+          nlp = spacy.load('en_core_web_sm')
+          doc = nlp('Apple is looking at buying U.K. startup for \$1 billion')
+          assert len(doc.ents) > 0, 'No entities found after upgrade'
+          print('Post-upgrade:', spacy.__version__, nlp.meta['name'], '@', nlp.meta['version'])
+          print('Entities:', [(ent.text, ent.label_) for ent in doc.ents])
+          "

.github/workflows/explosionbot.yml

@@ -6,6 +6,8 @@ on:
       - created
       - edited
 
+permissions: {}
+
 jobs:
   explosion-bot:
     if: github.repository_owner == 'explosion'
@@ -15,13 +17,15 @@ jobs:
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6
       - name: Install and run explosion-bot
         run: |
-          pip install git+https://${{ secrets.EXPLOSIONBOT_TOKEN }}@github.com/explosion/explosion-bot
+          git config --global url."https://x-access-token:${EXPLOSIONBOT_TOKEN}@github.com/".insteadOf "https://github.com/"
+          pip install git+https://github.com/explosion/explosion-bot
           python -m explosionbot
         env:
+          EXPLOSIONBOT_TOKEN: ${{ secrets.EXPLOSIONBOT_TOKEN }}
           INPUT_TOKEN: ${{ secrets.EXPLOSIONBOT_TOKEN }}
           INPUT_BK_TOKEN: ${{ secrets.BUILDKITE_SECRET }}
           ENABLED_COMMANDS: "test_gpu,test_slow,test_slow_gpu"

.github/workflows/gputests.yml.disabled

@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        branch: [master, main]
+        branch: [master, v4]
     if: github.repository_owner == 'explosion'
     runs-on: ubuntu-latest
     steps:

.github/workflows/issue-manager.yml

@@ -11,12 +11,16 @@ on:
     types:
       - labeled
 
+permissions: {}
+
 jobs:
   issue-manager:
+    permissions:
+      issues: write
     if: github.repository_owner == 'explosion'
     runs-on: ubuntu-latest
     steps:
-      - uses: tiangolo/issue-manager@0.4.0
+      - uses: tiangolo/issue-manager@4d1b7e05935a404dc8337d30bd23be46be8bb8e5  # 0.4.0
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           config: >

.github/workflows/lock.yml

@@ -16,7 +16,7 @@ jobs:
     if: github.repository_owner == 'explosion'
     runs-on: ubuntu-latest
     steps:
-      - uses: dessant/lock-threads@v4
+      - uses: dessant/lock-threads@1bf7ec25051fe7c00bdd17e6a7cf3d7bfb7dc771  # v5
         with:
           process-only: 'issues'
           issue-inactive-days: '30'

.github/workflows/publish_pypi.yml

@@ -0,0 +1,31 @@
+# The cibuildwheel action triggers on creation of a release, this
+# triggers on publication.
+# The expected workflow is to create a draft release and let the wheels
+# upload, and then hit 'publish', which uploads to PyPi.
+
+on:
+  release:
+    types:
+      - published
+
+permissions: {}
+
+jobs:
+  upload_pypi:
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/spacy
+    permissions:
+      id-token: write
+      contents: read
+    if: github.event_name == 'release' && github.event.action == 'published'
+    # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this)
+    # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - uses: robinraju/release-downloader@daf26c55d821e836577a15f77d86ddc078948b05  # v1
+        with:
+          tag: ${{ github.event.release.tag_name }}
+          fileName: '*'
+          out-file-path: 'dist'
+      - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1

.github/workflows/slowtests.yml.disabled

@@ -9,12 +9,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        branch: [master, main]
+        branch: [master, v4]
     if: github.repository_owner == 'explosion'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ matrix.branch }}
       - name: Get commits from past 24 hours

.github/workflows/spacy_universe_alert.yml

@@ -5,21 +5,16 @@ on:
     paths:
       - "website/meta/universe.json"
 
+permissions: {}
+
 jobs:
   build:
     if: github.repository_owner == 'explosion'
     runs-on: ubuntu-latest
 
     steps:
-      - name: Dump GitHub context
-        env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
-          PR_NUMBER: ${{github.event.number}}
-        run: |
-          echo "$GITHUB_CONTEXT"
-
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6
         with:
           python-version: '3.10'
       - name: Install Bernadette app dependency and send an alert
@@ -30,4 +25,4 @@ jobs:
         run: |
           pip install slack-sdk==3.17.2 aiohttp==3.8.1
           echo "$CHANNEL"
-          python .github/spacy_universe_alert.py "$GITHUB_CONTEXT"
+          echo "$GITHUB_CONTEXT" | python .github/spacy_universe_alert.py

.github/workflows/tests.yml

@@ -2,6 +2,8 @@ name: tests
 
 on:
   push:
+    tags-ignore:
+      - '**'
     branches-ignore:
       - "spacy.io"
       - "nightly.spacy.io"
@@ -10,7 +12,6 @@ on:
       - "*.md"
       - "*.mdx"
       - "website/**"
-      - ".github/workflows/**"
   pull_request:
     types: [opened, synchronize, reopened, edited]
     paths-ignore:
@@ -18,6 +19,9 @@ on:
       - "*.mdx"
       - "website/**"
 
+permissions:
+  contents: read
+
 jobs:
   validate:
     name: Validate
@@ -25,63 +29,40 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
 
       - name: Configure Python version
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6
         with:
-          python-version: "3.7"
-          architecture: x64
+          python-version: "3.10"
 
-      - name: black
+      - name: ruff format
         run: |
-          python -m pip install black -c requirements.txt
-          python -m black spacy --check
-      - name: isort
+          python -m pip install ruff -c requirements.txt
+          python -m ruff format spacy --check
+      - name: ruff isort
         run: |
-          python -m pip install isort -c requirements.txt
-          python -m isort spacy --check
-      - name: flake8
-        run: |
-          python -m pip install flake8==5.0.4
-          python -m flake8 spacy --count --select=E901,E999,F821,F822,F823,W605 --show-source --statistics
-      - name: cython-lint
-        run: |
-          python -m pip install cython-lint -c requirements.txt
-          # E501: line too log, W291: trailing whitespace, E266: too many leading '#' for block comment
-          cython-lint spacy --ignore E501,W291,E266
+          python -m ruff check spacy --select I
 
   tests:
     name: Test
     needs: Validate
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python_version: ["3.12"]
-        include:
-          - os: windows-latest
-            python_version: "3.7"
-          - os: macos-latest
-            python_version: "3.8"
-          - os: ubuntu-latest
-            python_version: "3.9"
-          - os: windows-latest
-            python_version: "3.10"
-          - os: macos-latest
-            python_version: "3.11"
+        python_version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
 
     runs-on: ${{ matrix.os }}
 
     steps:
       - name: Check out repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
 
       - name: Configure Python version
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6
         with:
           python-version: ${{ matrix.python_version }}
-          architecture: x64
 
       - name: Install dependencies
         run: |
@@ -115,7 +96,7 @@ jobs:
         shell: bash
 
       - name: Test import
-        run: python -W error -c "import spacy"
+        run: python -W error -W 'ignore:Core Pydantic V1:UserWarning:pydantic' -c "import spacy"
 
       - name: "Test download CLI"
         run: |
@@ -159,7 +140,9 @@ jobs:
       - name: "Test assemble CLI"
         run: |
           python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_sm'}; config.to_disk('ner_source_sm.cfg')"
-          PYTHONWARNINGS="error,ignore::DeprecationWarning" python -m spacy assemble ner_source_sm.cfg output_dir
+          python -m spacy assemble ner_source_sm.cfg output_dir
+        env:
+          PYTHONWARNINGS: "error,ignore::DeprecationWarning" 
         if: matrix.python_version == '3.9'
 
       - name: "Test assemble CLI vectors warning"
@@ -174,7 +157,7 @@ jobs:
 
       - name: "Run CPU tests"
         run: |
-          python -m pytest --pyargs spacy -W error
+          python -m pytest --pyargs spacy -W error -W 'ignore:Core Pydantic V1:UserWarning:pydantic'
         if: "!(startsWith(matrix.os, 'macos') && matrix.python_version == '3.11')"
 
       - name: "Run CPU tests with thinc-apple-ops"

.github/workflows/universe_validation.yml

@@ -13,6 +13,9 @@ on:
     paths:
       - "website/meta/universe.json"
 
+permissions:
+  contents: read
+
 jobs:
   validate:
     name: Validate
@@ -20,13 +23,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
 
       - name: Configure Python version
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6
         with:
           python-version: "3.7"
-          architecture: x64
 
       - name: Validate website/meta/universe.json
         run: |

.pre-commit-config.yaml

@@ -1,13 +1,7 @@
 repos:
--   repo: https://github.com/ambv/black
-    rev: 22.3.0
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.0
     hooks:
-    - id: black
-      language_version: python3.7
-      additional_dependencies: ['click==8.0.4']
--   repo: https://github.com/pycqa/flake8
-    rev: 5.0.4
-    hooks:
-    - id: flake8
-      args:
-        - "--config=setup.cfg"
+    - id: ruff
+      args: ['--fix']
+    - id: ruff-format

CONTRIBUTING.md

@@ -35,7 +35,7 @@ so that more people can benefit from it.
 
 When opening an issue, use a **descriptive title** and include your
 **environment** (operating system, Python version, spaCy version). Our
-[issue template](https://github.com/explosion/spaCy/issues/new) helps you
+[issue templates](https://github.com/explosion/spaCy/issues/new/choose) help you
 remember the most important details to include. If you've discovered a bug, you
 can also submit a [regression test](#fixing-bugs) straight away. When you're
 opening an issue to report the bug, simply refer to your pull request in the
@@ -449,8 +449,8 @@ and plugins in spaCy v3.0, and we can't wait to see what you build with it!
   [`spacy`](https://github.com/topics/spacy?o=desc&s=stars) and
   [`spacy-extensions`](https://github.com/topics/spacy-extension?o=desc&s=stars)
   to make it easier to find. Those are also the topics we're linking to from the
-  spaCy website. If you're sharing your project on Twitter, feel free to tag
-  [@spacy_io](https://twitter.com/spacy_io) so we can check it out.
+  spaCy website. If you're sharing your project on X, feel free to tag
+  [@spacy_io](https://x.com/spacy_io) so we can check it out.
 
 - Once your extension is published, you can open a
   [PR](https://github.com/explosion/spaCy/pulls) to suggest it for the

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (C) 2016-2023 ExplosionAI GmbH, 2016 spaCy GmbH, 2015 Matthew Honnibal
+Copyright (C) 2016-2024 ExplosionAI GmbH, 2016 spaCy GmbH, 2015 Matthew Honnibal
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

MANIFEST.in

@@ -4,5 +4,6 @@ include README.md
 include pyproject.toml
 include spacy/py.typed
 recursive-include spacy/cli *.yml
+recursive-include spacy/tests *.json
 recursive-include licenses *
 recursive-exclude spacy *.cpp

README.md

@@ -16,7 +16,7 @@ model packaging, deployment and workflow management. spaCy is commercial
 open-source software, released under the
 [MIT license](https://github.com/explosion/spaCy/blob/master/LICENSE).
 
-💫 **Version 3.7 out now!**
+💫 **Version 3.8 out now!**
 [Check out the release notes here.](https://github.com/explosion/spaCy/releases)
 
 [![tests](https://github.com/explosion/spaCy/actions/workflows/tests.yml/badge.svg)](https://github.com/explosion/spaCy/actions/workflows/tests.yml)
@@ -28,7 +28,6 @@ open-source software, released under the
 <br />
 [![PyPi downloads](https://static.pepy.tech/personalized-badge/spacy?period=total&units=international_system&left_color=grey&right_color=orange&left_text=pip%20downloads)](https://pypi.org/project/spacy/)
 [![Conda downloads](https://img.shields.io/conda/dn/conda-forge/spacy?label=conda%20downloads)](https://anaconda.org/conda-forge/spacy)
-[![spaCy on Twitter](https://img.shields.io/twitter/follow/spacy_io.svg?style=social&label=Follow)](https://twitter.com/spacy_io)
 
 ## 📖 Documentation
 
@@ -47,6 +46,7 @@ open-source software, released under the
 | 👩‍🏫 **[Online Course]**                                                                                                                                                                                                    | Learn spaCy in this free and interactive online course.                                                                                                                                                                                                                                                                                      |
 | 📰 **[Blog]**                                                                                                                                                                                                             | Read about current spaCy and Prodigy development, releases, talks and more from Explosion.                                                                                                                                                                                                                 |
 | 📺 **[Videos]**                                                                                                                                                                                                           | Our YouTube channel with video tutorials, talks and more.                                                                                                                                                                                                                                                                                    |
+| 🔴 **[Live Stream]**                                                                                                                                                                                                       | Join Matt as he works on spaCy and chat about NLP.                                                                                                                                                                                                                                                                         |
 | 🛠 **[Changelog]**                                                                                                                                                                                                         | Changes and version history.                                                                                                                                                                                                                                                                                                                 |
 | 💝 **[Contribute]**                                                                                                                                                                                                       | How to contribute to the spaCy project and code base.                                                                                                                                                                                                                                                                                        |
 | 👕 **[Swag]**                                                                                                                                                                                                             | Support us and our work with unique, custom-designed swag!                                                                                                                                                                                                                                                                                   |
@@ -62,6 +62,7 @@ open-source software, released under the
 [universe]: https://spacy.io/universe
 [spacy vs code extension]: https://github.com/explosion/spacy-vscode
 [videos]: https://www.youtube.com/c/ExplosionAI
+[live stream]: https://www.youtube.com/playlist?list=PLBmcuObd5An5_iAxNYLJa_xWmNzsYce8c
 [online course]: https://course.spacy.io
 [blog]: https://explosion.ai
 [project templates]: https://github.com/explosion/projects
@@ -79,13 +80,14 @@ more people can benefit from it.
 | Type                            | Platforms                               |
 | ------------------------------- | --------------------------------------- |
 | 🚨 **Bug Reports**              | [GitHub Issue Tracker]                  |
-| 🎁 **Feature Requests & Ideas** | [GitHub Discussions]                    |
+| 🎁 **Feature Requests & Ideas** | [GitHub Discussions] · [Live Stream]    |
 | 👩‍💻 **Usage Questions**          | [GitHub Discussions] · [Stack Overflow] |
-| 🗯 **General Discussion**        | [GitHub Discussions]                    |
+| 🗯 **General Discussion**        | [GitHub Discussions] · [Live Stream]   |
 
 [github issue tracker]: https://github.com/explosion/spaCy/issues
 [github discussions]: https://github.com/explosion/spaCy/discussions
 [stack overflow]: https://stackoverflow.com/questions/tagged/spacy
+[live stream]: https://www.youtube.com/playlist?list=PLBmcuObd5An5_iAxNYLJa_xWmNzsYce8c
 
 ## Features
 
@@ -115,7 +117,7 @@ For detailed installation instructions, see the
 
 - **Operating system**: macOS / OS X · Linux · Windows (Cygwin, MinGW, Visual
   Studio)
-- **Python version**: Python 3.7+ (only 64 bit)
+- **Python version**: Python >=3.7, <3.13 (only 64 bit)
 - **Package managers**: [pip] · [conda] (via `conda-forge`)
 
 [pip]: https://pypi.org/project/spacy/

RELEASE_NOTES.md

@@ -0,0 +1,5 @@
+Bug fix for model downloading in environments without pip on PATH
+
+## Fixes
+
+- Fix `spacy download` failing in environments where `pip` is not on PATH but is available as a Python module (e.g., some virtual environments and containers)

bin/release.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Insist repository is clean
+git diff-index --quiet HEAD
+
+version=$(grep "__version__ = " spacy/about.py)
+version=${version/__version__ = }
+version=${version/\'/}
+version=${version/\'/}
+version=${version/\"/}
+version=${version/\"/}
+
+echo "Pushing release-v"$version
+
+git tag -d release-v$version || true
+git push origin :release-v$version || true
+git tag release-v$version
+git push origin release-v$version

build-constraints.txt

@@ -1,6 +1,2 @@
 # build version constraints for use with wheelwright
-numpy==1.15.0; python_version=='3.7' and platform_machine!='aarch64'
-numpy==1.19.2; python_version=='3.7' and platform_machine=='aarch64'
-numpy==1.17.3; python_version=='3.8' and platform_machine!='aarch64'
-numpy==1.19.2; python_version=='3.8' and platform_machine=='aarch64'
-numpy>=1.25.0; python_version>='3.9'
+numpy>=2.0.0,<3.0.0

pyproject.toml

@@ -1,15 +1,76 @@
 [build-system]
 requires = [
     "setuptools",
-    "cython>=0.25,<3.0",
+    "cython>=3.0,<4.0",
     "cymem>=2.0.2,<2.1.0",
     "preshed>=3.0.2,<3.1.0",
     "murmurhash>=0.28.0,<1.1.0",
-    "thinc>=8.2.2,<8.3.0",
-    "numpy>=1.15.0; python_version < '3.9'",
-    "numpy>=1.25.0; python_version >= '3.9'",
+    "thinc>=8.3.12,<8.4.0",
+    "numpy>=2.0.0,<3.0.0"
 ]
 build-backend = "setuptools.build_meta"
 
-[tool.isort]
-profile = "black"
+[tool.cibuildwheel]
+build = "*"
+skip = "cp39* *-win32 *i686* cp3??t-* *cp310-win_arm64"
+test-skip = ""
+
+archs = ["native"]
+
+build-frontend = "default"
+config-settings = {}
+dependency-versions = "pinned"
+environment = { PIP_CONSTRAINT = "build-constraints.txt" }
+
+environment-pass = []
+build-verbosity = 0
+
+before-all = "curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain stable"
+before-build = "pip install -r requirements.txt && python setup.py clean"
+repair-wheel-command = ""
+
+test-command = ""
+before-test = ""
+test-requires = []
+test-extras = []
+
+container-engine = "docker"
+
+# numpy >=2.3 only ships manylinux_2_28 wheels, so the build container must
+# be at least that; i686 keeps manylinux2014 (no 2_28 image) but isn't built
+manylinux-x86_64-image = "manylinux_2_28"
+manylinux-i686-image = "manylinux2014"
+manylinux-aarch64-image = "manylinux_2_28"
+manylinux-ppc64le-image = "manylinux2014"
+manylinux-s390x-image = "manylinux2014"
+manylinux-pypy_x86_64-image = "manylinux2014"
+manylinux-pypy_i686-image = "manylinux2014"
+manylinux-pypy_aarch64-image = "manylinux2014"
+
+musllinux-x86_64-image = "musllinux_1_2"
+musllinux-i686-image = "musllinux_1_2"
+musllinux-aarch64-image = "musllinux_1_2"
+musllinux-ppc64le-image = "musllinux_1_2"
+musllinux-s390x-image = "musllinux_1_2"
+
+[tool.cibuildwheel.linux]
+repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}"
+
+[tool.cibuildwheel.macos]
+repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}"
+
+[tool.cibuildwheel.windows]
+
+[tool.cibuildwheel.pyodide]
+
+
+[tool.ruff]
+line-length = 88
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "C", "B", "B9"]
+ignore = ["E203", "E266", "E501", "E731", "E741", "F541"]
+
+[tool.ruff.lint.isort]
+combine-as-imports = true
+split-on-trailing-comma = true

requirements.txt

@@ -3,40 +3,38 @@ spacy-legacy>=3.0.11,<3.1.0
 spacy-loggers>=1.0.0,<2.0.0
 cymem>=2.0.2,<2.1.0
 preshed>=3.0.2,<3.1.0
-thinc>=8.2.2,<8.3.0
-ml_datasets>=0.2.0,<0.3.0
+thinc>=8.3.12,<8.4.0
+ml_datasets>=0.2.1,<0.3.0
 murmurhash>=0.28.0,<1.1.0
 wasabi>=0.9.1,<1.2.0
-srsly>=2.4.3,<3.0.0
+srsly>=2.5.3,<3.0.0
 catalogue>=2.0.6,<2.1.0
-typer>=0.3.0,<0.10.0
-smart-open>=5.2.1,<7.0.0
-weasel>=0.1.0,<0.4.0
+typer>=0.3.0,<1.0.0
+click>=8.2.1,<9.0.0
+weasel>=1.0.0,<2.0.0
 # Third party dependencies
-numpy>=1.15.0; python_version < "3.9"
-numpy>=1.19.0; python_version >= "3.9"
+numpy>=2.0.0,<3.0.0
 requests>=2.13.0,<3.0.0
 tqdm>=4.38.0,<5.0.0
-pydantic>=1.7.4,!=1.8,!=1.8.1,<3.0.0
+pydantic>=2.0.0,<3.0.0
 jinja2
-langcodes>=3.2.0,<4.0.0
 # Official Python utilities
 setuptools
 packaging>=20.0
-typing_extensions>=3.7.4.1,<4.5.0; python_version < "3.8"
 # Development dependencies
 pre-commit>=2.13.0
-cython>=0.25,<3.0
+cython>=3.0,<4.0
 pytest>=5.2.0,!=7.1.0
 pytest-timeout>=1.3.0,<2.0.0
 mock>=2.0.0,<3.0.0
-flake8>=3.8.0,<6.0.0
-hypothesis>=3.27.0,<7.0.0
-mypy>=1.5.0,<1.6.0; platform_machine != "aarch64" and python_version >= "3.8"
+# hypothesis >=6.156 ships a Rust extension with no win_arm64 wheels, which
+# breaks wheel builds on windows-11-arm (sdist needs maturin)
+hypothesis>=3.27.0,<6.156.0
+mypy>=1.20.2,<1.21.0; platform_machine != "aarch64" and python_version >= "3.8"
 types-mock>=0.1.1
 types-setuptools>=57.0.0
 types-requests
 types-setuptools>=57.0.0
-black==22.3.0
+ruff>=0.9.0
 cython-lint>=0.15.0
-isort>=5.0,<6.0
+confection>=1.3.2,<2.0.0

setup.cfg

@@ -17,11 +17,12 @@ classifiers =
     Operating System :: Microsoft :: Windows
     Programming Language :: Cython
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.7
-    Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
+    Programming Language :: Python :: 3.12
+    Programming Language :: Python :: 3.13
+    Programming Language :: Python :: 3.14
     Topic :: Scientific/Engineering
 project_urls =
     Release notes = https://github.com/explosion/spaCy/releases
@@ -30,18 +31,18 @@ project_urls =
 [options]
 zip_safe = false
 include_package_data = true
-python_requires = >=3.7
+python_requires = >=3.9,<3.15
 # NOTE: This section is superseded by pyproject.toml and will be removed in
 # spaCy v4
 setup_requires =
-    cython>=0.25,<3.0
-    numpy>=1.15.0; python_version < "3.9"
-    numpy>=1.19.0; python_version >= "3.9"
+    cython>=3.0,<4.0
+    numpy>=2.0.0,<3.0.0; python_version < "3.9"
+    numpy>=2.0.0,<3.0.0; python_version >= "3.9"
     # We also need our Cython packages here to compile against
     cymem>=2.0.2,<2.1.0
     preshed>=3.0.2,<3.1.0
     murmurhash>=0.28.0,<1.1.0
-    thinc>=8.2.2,<8.3.0
+    thinc>=8.3.12,<8.4.0
 install_requires =
     # Our libraries
     spacy-legacy>=3.0.11,<3.1.0
@@ -49,25 +50,24 @@ install_requires =
     murmurhash>=0.28.0,<1.1.0
     cymem>=2.0.2,<2.1.0
     preshed>=3.0.2,<3.1.0
-    thinc>=8.2.2,<8.3.0
+    thinc>=8.3.12,<8.4.0
     wasabi>=0.9.1,<1.2.0
-    srsly>=2.4.3,<3.0.0
+    srsly>=2.5.3,<3.0.0
     catalogue>=2.0.6,<2.1.0
-    weasel>=0.1.0,<0.4.0
+    weasel>=1.0.0,<2.0.0
+    confection>=1.3.2,<2.0.0
     # Third-party dependencies
-    typer>=0.3.0,<0.10.0
-    smart-open>=5.2.1,<7.0.0
+    typer>=0.3.0,<1.0.0
+    click>=8.2.1,<9.0.0
     tqdm>=4.38.0,<5.0.0
     numpy>=1.15.0; python_version < "3.9"
     numpy>=1.19.0; python_version >= "3.9"
     requests>=2.13.0,<3.0.0
-    pydantic>=1.7.4,!=1.8,!=1.8.1,<3.0.0
+    pydantic>=2.0.0,<3.0.0
     jinja2
     # Official Python utilities
     setuptools
     packaging>=20.0
-    typing_extensions>=3.7.4.1,<4.5.0; python_version < "3.8"
-    langcodes>=3.2.0,<4.0.0
 
 [options.entry_points]
 console_scripts =
@@ -117,7 +117,7 @@ cuda12x =
 cuda-autodetect =
     cupy-wheel>=11.0.0,<13.0.0
 apple =
-    thinc-apple-ops>=0.1.0.dev0,<1.0.0
+    thinc-apple-ops>=1.0.0,<2.0.0
 # Language tokenizers with external dependencies
 ja =
     sudachipy>=0.5.2,!=0.6.1
@@ -133,20 +133,13 @@ universal = false
 [sdist]
 formats = gztar
 
-[flake8]
-ignore = E203, E266, E501, E731, W503, E741, F541
-max-line-length = 80
-select = B,C,E,F,W,T4,B9
-exclude =
-    .env,
-    .git,
-    __pycache__,
-    _tokenizer_exceptions_list.py,
-
 [tool:pytest]
 markers =
     slow: mark a test as slow
     issue: reference specific issue
+filterwarnings =
+    error
+    ignore:Core Pydantic V1:UserWarning:pydantic
 
 [mypy]
 ignore_missing_imports = True

setup.py

@@ -82,9 +82,9 @@ COMPILER_DIRECTIVES = {
 }
 # Files to copy into the package that are otherwise not included
 COPY_FILES = {
-    ROOT / "setup.cfg": PACKAGE_ROOT / "tests" / "package",
-    ROOT / "pyproject.toml": PACKAGE_ROOT / "tests" / "package",
-    ROOT / "requirements.txt": PACKAGE_ROOT / "tests" / "package",
+    ROOT / "setup.cfg": PACKAGE_ROOT / "tests" / "package" / "test.cfg",
+    ROOT / "pyproject.toml": PACKAGE_ROOT / "tests" / "package" / "test.toml",
+    ROOT / "requirements.txt": PACKAGE_ROOT / "tests" / "package" / "test.txt",
 }
 
 
@@ -173,10 +173,10 @@ def setup_package():
         about = {}
         exec(f.read(), about)
 
-    for copy_file, target_dir in COPY_FILES.items():
+    for copy_file, target_file in COPY_FILES.items():
         if copy_file.exists():
-            shutil.copy(str(copy_file), str(target_dir))
-            print(f"Copied {copy_file} -> {target_dir}")
+            shutil.copyfile(str(copy_file), str(target_file))
+            print(f"Copied {copy_file} -> {target_file}")
 
     include_dirs = [
         numpy.get_include(),

spacy/__init__.py

@@ -10,16 +10,39 @@ setup_default_warnings()  # noqa: E402
 # These are imported as part of the API
 from thinc.api import Config, prefer_gpu, require_cpu, require_gpu  # noqa: F401
 
-from . import pipeline  # noqa: F401
-from . import util
+from . import (
+    pipeline,  # noqa: F401
+    util,
+)
 from .about import __version__  # noqa: F401
 from .cli.info import info  # noqa: F401
 from .errors import Errors
 from .glossary import explain  # noqa: F401
 from .language import Language
+from .registrations import REGISTRY_POPULATED, populate_registry
+
+# Rebuild pydantic v2 schemas that use forward references to Language/Vocab
+from .schemas import (  # noqa: F401
+    ConfigSchema,
+    ConfigSchemaInit,
+    ConfigSchemaNlp,
+    ConfigSchemaPretrain,
+    ConfigSchemaTraining,
+)
+from .training import Example  # noqa: F401
 from .util import logger, registry  # noqa: F401
 from .vocab import Vocab
 
+_rebuild_ns = {"Language": Language, "Vocab": Vocab, "Example": Example}
+for _schema in (
+    ConfigSchemaTraining,
+    ConfigSchemaNlp,
+    ConfigSchemaPretrain,
+    ConfigSchemaInit,
+    ConfigSchema,
+):
+    _schema.model_rebuild(_types_namespace=_rebuild_ns)  # type: ignore[attr-defined]
+
 if sys.maxunicode == 65535:
     raise SystemError(Errors.E130)
 

spacy/about.py

@@ -1,5 +1,5 @@
 # fmt: off
 __title__ = "spacy"
-__version__ = "3.7.3"
+__version__ = "3.8.15"
 __download_url__ = "https://github.com/explosion/spacy-models/releases/download"
 __compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"

spacy/cli/__init__.py

@@ -1,5 +1,7 @@
 from wasabi import msg
 
+# Needed for testing
+from . import download as download_module  # noqa: F401
 from ._util import app, setup_cli  # noqa: F401
 from .apply import apply  # noqa: F401
 from .assemble import assemble_cli  # noqa: F401

spacy/cli/_util.py

@@ -1,15 +1,11 @@
-import hashlib
 import os
-import shutil
 import sys
 from configparser import InterpolationError
 from contextlib import contextmanager
 from pathlib import Path
 from typing import (
-    TYPE_CHECKING,
     Any,
     Dict,
-    Iterable,
     List,
     Optional,
     Tuple,
@@ -20,23 +16,18 @@ from typing import (
 import srsly
 import typer
 from click import NoSuchOption
-from click.parser import split_arg_string
-from thinc.api import Config, ConfigValidationError, require_gpu
+from click.shell_completion import split_arg_string
+from thinc.api import ConfigValidationError, require_gpu
 from thinc.util import gpu_is_available
 from typer.main import get_command
 from wasabi import Printer, msg
 from weasel import app as project_cli
 
-from .. import about
 from ..compat import Literal
-from ..schemas import validate
 from ..util import (
     ENV_VARS,
-    SimpleFrozenDict,
     import_file,
-    is_compatible_version,
     logger,
-    make_tempdir,
     registry,
     run_command,
 )
@@ -68,7 +59,7 @@ INIT_HELP = """Commands for initializing configs and pipeline packages."""
 Arg = typer.Argument
 Opt = typer.Option
 
-app = typer.Typer(name=NAME, help=HELP)
+app = typer.Typer(name=NAME, help=HELP, rich_markup_mode=None)
 benchmark_cli = typer.Typer(name="benchmark", help=BENCHMARK_HELP, no_args_is_help=True)
 debug_cli = typer.Typer(name="debug", help=DEBUG_HELP, no_args_is_help=True)
 init_cli = typer.Typer(name="init", help=INIT_HELP, no_args_is_help=True)
@@ -225,13 +216,11 @@ def get_git_version(
 
 
 @overload
-def string_to_list(value: str, intify: Literal[False] = ...) -> List[str]:
-    ...
+def string_to_list(value: str, intify: Literal[False] = ...) -> List[str]: ...
 
 
 @overload
-def string_to_list(value: str, intify: Literal[True]) -> List[int]:
-    ...
+def string_to_list(value: str, intify: Literal[True]) -> List[int]: ...
 
 
 def string_to_list(value: str, intify: bool = False) -> Union[List[str], List[int]]:

spacy/cli/apply.py

@@ -22,7 +22,7 @@ to be grabbed ("text" by default)."""
 
 out_help = "Path to save the resulting .spacy file"
 code_help = (
-    "Path to Python file with additional " "code (registered functions) to be imported"
+    "Path to Python file with additional code (registered functions) to be imported"
 )
 gold_help = "Use gold preprocessing provided in the .spacy files"
 force_msg = (
@@ -72,11 +72,15 @@ def apply_cli(
     data_path: Path = Arg(..., help=path_help, exists=True),
     output_file: Path = Arg(..., help=out_help, dir_okay=False),
     code_path: Optional[Path] = Opt(None, "--code", "-c", help=code_help),
-    text_key: str = Opt("text", "--text-key", "-tk", help="Key containing text string for JSONL"),
-    force_overwrite: bool = Opt(False, "--force", "-F", help="Force overwriting the output file"),
+    text_key: str = Opt(
+        "text", "--text-key", "-tk", help="Key containing text string for JSONL"
+    ),
+    force_overwrite: bool = Opt(
+        False, "--force", "-F", help="Force overwriting the output file"
+    ),
     use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU."),
     batch_size: int = Opt(1, "--batch-size", "-b", help="Batch size."),
-    n_process: int = Opt(1, "--n-process", "-n", help="number of processors to use.")
+    n_process: int = Opt(1, "--n-process", "-n", help="number of processors to use."),
 ):
     """
     Apply a trained pipeline to documents to get predictions.
@@ -114,8 +118,7 @@ def apply(
     if len(paths) == 0:
         docbin.to_disk(output_file)
         msg.warn(
-            "Did not find data to process,"
-            f" {data_path} seems to be an empty directory."
+            f"Did not find data to process, {data_path} seems to be an empty directory."
         )
         return
     nlp = load_model(model)

spacy/cli/assemble.py

@@ -24,10 +24,25 @@ from ._util import (
 def assemble_cli(
     # fmt: off
     ctx: typer.Context,  # This is only used to read additional arguments
-    config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
-    output_path: Path = Arg(..., help="Output directory to store assembled pipeline in"),
-    code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
-    verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
+    config_path: Path = Arg(
+        ..., help="Path to config file", exists=True, allow_dash=True
+    ),
+    output_path: Path = Arg(
+        ..., help="Output directory to store assembled pipeline in"
+    ),
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
+    verbose: bool = Opt(
+        False,
+        "--verbose",
+        "-V",
+        "-VV",
+        help="Display more information for debugging purposes",
+    ),
     # fmt: on
 ):
     """

spacy/cli/benchmark_speed.py

@@ -24,13 +24,29 @@ def benchmark_speed_cli(
     # fmt: off
     ctx: typer.Context,
     model: str = Arg(..., help="Model name or path"),
-    data_path: Path = Arg(..., help="Location of binary evaluation data in .spacy format", exists=True),
-    batch_size: Optional[int] = Opt(None, "--batch-size", "-b", min=1, help="Override the pipeline batch size"),
+    data_path: Path = Arg(
+        ..., help="Location of binary evaluation data in .spacy format", exists=True
+    ),
+    batch_size: Optional[int] = Opt(
+        None, "--batch-size", "-b", min=1, help="Override the pipeline batch size"
+    ),
     no_shuffle: bool = Opt(False, "--no-shuffle", help="Do not shuffle benchmark data"),
     use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
-    n_batches: int = Opt(50, "--batches", help="Minimum number of batches to benchmark", min=30,),
-    warmup_epochs: int = Opt(3, "--warmup", "-w", min=0, help="Number of iterations over the data for warmup"),
-    code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
+    n_batches: int = Opt(
+        50,
+        "--batches",
+        help="Minimum number of batches to benchmark",
+        min=30,
+    ),
+    warmup_epochs: int = Opt(
+        3, "--warmup", "-w", min=0, help="Number of iterations over the data for warmup"
+    ),
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
     # fmt: on
 ):
     """
@@ -151,7 +167,7 @@ def print_mean_with_ci(sample: numpy.ndarray):
     low = bootstrap_means[int(len(bootstrap_means) * 0.025)]
     high = bootstrap_means[int(len(bootstrap_means) * 0.975)]
 
-    print(f"Mean: {mean:.1f} words/s (95% CI: {low-mean:.1f} +{high-mean:.1f})")
+    print(f"Mean: {mean:.1f} words/s (95% CI: {low - mean:.1f} +{high - mean:.1f})")
 
 
 def print_outliers(sample: numpy.ndarray):

spacy/cli/convert.py

@@ -48,17 +48,47 @@ class FileTypes(str, Enum):
 def convert_cli(
     # fmt: off
     input_path: str = Arg(..., help="Input file or directory", exists=True),
-    output_dir: Path = Arg("-", help="Output directory. '-' for stdout.", allow_dash=True, exists=True),
-    file_type: FileTypes = Opt("spacy", "--file-type", "-t", help="Type of data to produce"),
-    n_sents: int = Opt(1, "--n-sents", "-n", help="Number of sentences per doc (0 to disable)"),
-    seg_sents: bool = Opt(False, "--seg-sents", "-s", help="Segment sentences (for -c ner)"),
-    model: Optional[str] = Opt(None, "--model", "--base", "-b", help="Trained spaCy pipeline for sentence segmentation to use as base (for --seg-sents)"),
-    morphology: bool = Opt(False, "--morphology", "-m", help="Enable appending morphology to tags"),
-    merge_subtokens: bool = Opt(False, "--merge-subtokens", "-T", help="Merge CoNLL-U subtokens"),
-    converter: str = Opt(AUTO, "--converter", "-c", help=f"Converter: {tuple(CONVERTERS.keys())}"),
-    ner_map: Optional[Path] = Opt(None, "--ner-map", "-nm", help="NER tag mapping (as JSON-encoded dict of entity types)", exists=True),
-    lang: Optional[str] = Opt(None, "--lang", "-l", help="Language (if tokenizer required)"),
-    concatenate: bool = Opt(None, "--concatenate", "-C", help="Concatenate output to a single file"),
+    output_dir: Path = Arg(
+        "-", help="Output directory. '-' for stdout.", allow_dash=True, exists=True
+    ),
+    file_type: FileTypes = Opt(
+        "spacy", "--file-type", "-t", help="Type of data to produce"
+    ),
+    n_sents: int = Opt(
+        1, "--n-sents", "-n", help="Number of sentences per doc (0 to disable)"
+    ),
+    seg_sents: bool = Opt(
+        False, "--seg-sents", "-s", help="Segment sentences (for -c ner)"
+    ),
+    model: Optional[str] = Opt(
+        None,
+        "--model",
+        "--base",
+        "-b",
+        help="Trained spaCy pipeline for sentence segmentation to use as base (for --seg-sents)",
+    ),
+    morphology: bool = Opt(
+        False, "--morphology", "-m", help="Enable appending morphology to tags"
+    ),
+    merge_subtokens: bool = Opt(
+        False, "--merge-subtokens", "-T", help="Merge CoNLL-U subtokens"
+    ),
+    converter: str = Opt(
+        AUTO, "--converter", "-c", help=f"Converter: {tuple(CONVERTERS.keys())}"
+    ),
+    ner_map: Optional[Path] = Opt(
+        None,
+        "--ner-map",
+        "-nm",
+        help="NER tag mapping (as JSON-encoded dict of entity types)",
+        exists=True,
+    ),
+    lang: Optional[str] = Opt(
+        None, "--lang", "-l", help="Language (if tokenizer required)"
+    ),
+    concatenate: bool = Opt(
+        None, "--concatenate", "-C", help="Concatenate output to a single file"
+    ),
     # fmt: on
 ):
     """

spacy/cli/debug_config.py

@@ -26,10 +26,28 @@ from ._util import (
 def debug_config_cli(
     # fmt: off
     ctx: typer.Context,  # This is only used to read additional arguments
-    config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
-    code_path: Optional[Path] = Opt(None, "--code-path", "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
-    show_funcs: bool = Opt(False, "--show-functions", "-F", help="Show an overview of all registered functions used in the config and where they come from (modules, files etc.)"),
-    show_vars: bool = Opt(False, "--show-variables", "-V", help="Show an overview of all variables referenced in the config and their values. This will also reflect variables overwritten on the CLI.")
+    config_path: Path = Arg(
+        ..., help="Path to config file", exists=True, allow_dash=True
+    ),
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code-path",
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
+    show_funcs: bool = Opt(
+        False,
+        "--show-functions",
+        "-F",
+        help="Show an overview of all registered functions used in the config and where they come from (modules, files etc.)",
+    ),
+    show_vars: bool = Opt(
+        False,
+        "--show-variables",
+        "-V",
+        help="Show an overview of all variables referenced in the config and their values. This will also reflect variables overwritten on the CLI.",
+    ),
     # fmt: on
 ):
     """Debug a config file and show validation errors. The command will
@@ -64,10 +82,10 @@ def debug_config(
         config = nlp.config.interpolate()
     msg.divider("Config validation for [initialize]")
     with show_validation_error(config_path):
-        T = registry.resolve(config["initialize"], schema=ConfigSchemaInit)
+        T = registry.resolve(config["initialize"], schema=ConfigSchemaInit)  # type: ignore[arg-type]
     msg.divider("Config validation for [training]")
     with show_validation_error(config_path):
-        T = registry.resolve(config["training"], schema=ConfigSchemaTraining)
+        T = registry.resolve(config["training"], schema=ConfigSchemaTraining)  # type: ignore[arg-type]
         dot_names = [T["train_corpus"], T["dev_corpus"]]
         util.resolve_dot_names(config, dot_names)
     msg.good("Config is valid")

spacy/cli/debug_data.py

@@ -71,11 +71,28 @@ SPAN_LENGTH_THRESHOLD_PERCENTAGE = 90
 def debug_data_cli(
     # fmt: off
     ctx: typer.Context,  # This is only used to read additional arguments
-    config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
-    code_path: Optional[Path] = Opt(None, "--code-path", "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
-    ignore_warnings: bool = Opt(False, "--ignore-warnings", "-IW", help="Ignore warnings, only show stats and errors"),
-    verbose: bool = Opt(False, "--verbose", "-V", help="Print additional information and explanations"),
-    no_format: bool = Opt(False, "--no-format", "-NF", help="Don't pretty-print the results"),
+    config_path: Path = Arg(
+        ..., help="Path to config file", exists=True, allow_dash=True
+    ),
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code-path",
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
+    ignore_warnings: bool = Opt(
+        False,
+        "--ignore-warnings",
+        "-IW",
+        help="Ignore warnings, only show stats and errors",
+    ),
+    verbose: bool = Opt(
+        False, "--verbose", "-V", help="Print additional information and explanations"
+    ),
+    no_format: bool = Opt(
+        False, "--no-format", "-NF", help="Don't pretty-print the results"
+    ),
     # fmt: on
 ):
     """
@@ -120,7 +137,7 @@ def debug_data(
         cfg = util.load_config(config_path, overrides=config_overrides)
         nlp = util.load_model_from_config(cfg)
         config = nlp.config.interpolate()
-        T = registry.resolve(config["training"], schema=ConfigSchemaTraining)
+        T = registry.resolve(config["training"], schema=ConfigSchemaTraining)  # type: ignore[arg-type]
     # Use original config here, not resolved version
     sourced_components = get_sourced_components(cfg)
     frozen_components = T["frozen_components"]
@@ -562,7 +579,7 @@ def debug_data(
 
     if "morphologizer" in factory_names:
         msg.divider("Morphologizer (POS+Morph)")
-        label_list = [label for label in gold_train_data["morphs"]]
+        label_list = tuple(gold_train_data["morphs"])
         model_labels = _get_labels_from_model(nlp, "morphologizer")
         msg.info(f"{len(label_list)} label(s) in train data")
         labels = set(label_list)
@@ -708,7 +725,7 @@ def debug_data(
         if len(dev_not_train) != 0:
             pct = len(dev_not_train) / len(trees_dev)
             msg.info(
-                f"{len(dev_not_train)} lemmatizer trees ({pct*100:.1f}% of dev trees)"
+                f"{len(dev_not_train)} lemmatizer trees ({pct * 100:.1f}% of dev trees)"
                 " were found exclusively in the dev data."
             )
         else:
@@ -968,16 +985,14 @@ def _compile_gold(
 
 
 @overload
-def _format_labels(labels: Iterable[str], counts: Literal[False] = False) -> str:
-    ...
+def _format_labels(labels: Iterable[str], counts: Literal[False] = False) -> str: ...
 
 
 @overload
 def _format_labels(
     labels: Iterable[Tuple[str, int]],
     counts: Literal[True],
-) -> str:
-    ...
+) -> str: ...
 
 
 def _format_labels(

spacy/cli/debug_diff.py

@@ -2,11 +2,10 @@ from pathlib import Path
 from typing import Optional
 
 import typer
-from thinc.api import Config
 from wasabi import MarkdownRenderer, Printer, diff_strings
 
 from ..util import load_config
-from ._util import Arg, Opt, debug_cli, parse_config_overrides, show_validation_error
+from ._util import Arg, Opt, debug_cli, show_validation_error
 from .init_config import Optimizations, init_config
 
 
@@ -17,12 +16,36 @@ from .init_config import Optimizations, init_config
 def debug_diff_cli(
     # fmt: off
     ctx: typer.Context,
-    config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
-    compare_to: Optional[Path] = Opt(None, help="Path to a config file to diff against, or `None` to compare against default settings", exists=True, allow_dash=True),
-    optimize: Optimizations = Opt(Optimizations.efficiency.value, "--optimize", "-o", help="Whether the user config was optimized for efficiency or accuracy. Only relevant when comparing against the default config."),
-    gpu: bool = Opt(False, "--gpu", "-G", help="Whether the original config can run on a GPU. Only relevant when comparing against the default config."),
-    pretraining: bool = Opt(False, "--pretraining", "--pt", help="Whether to compare on a config with pretraining involved. Only relevant when comparing against the default config."),
-    markdown: bool = Opt(False, "--markdown", "-md", help="Generate Markdown for GitHub issues")
+    config_path: Path = Arg(
+        ..., help="Path to config file", exists=True, allow_dash=True
+    ),
+    compare_to: Optional[Path] = Opt(
+        None,
+        help="Path to a config file to diff against, or `None` to compare against default settings",
+        exists=True,
+        allow_dash=True,
+    ),
+    optimize: Optimizations = Opt(
+        Optimizations.efficiency.value,
+        "--optimize",
+        "-o",
+        help="Whether the user config was optimized for efficiency or accuracy. Only relevant when comparing against the default config.",
+    ),
+    gpu: bool = Opt(
+        False,
+        "--gpu",
+        "-G",
+        help="Whether the original config can run on a GPU. Only relevant when comparing against the default config.",
+    ),
+    pretraining: bool = Opt(
+        False,
+        "--pretraining",
+        "--pt",
+        help="Whether to compare on a config with pretraining involved. Only relevant when comparing against the default config.",
+    ),
+    markdown: bool = Opt(
+        False, "--markdown", "-md", help="Generate Markdown for GitHub issues"
+    ),
     # fmt: on
 ):
     """Show a diff of a config file with respect to spaCy's defaults or another config file. If

spacy/cli/debug_model.py

@@ -36,18 +36,26 @@ from ._util import (
 def debug_model_cli(
     # fmt: off
     ctx: typer.Context,  # This is only used to read additional arguments
-    config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
-    component: str = Arg(..., help="Name of the pipeline component of which the model should be analysed"),
-    layers: str = Opt("", "--layers", "-l", help="Comma-separated names of layer IDs to print"),
+    config_path: Path = Arg(
+        ..., help="Path to config file", exists=True, allow_dash=True
+    ),
+    component: str = Arg(
+        ..., help="Name of the pipeline component of which the model should be analysed"
+    ),
+    layers: str = Opt(
+        "", "--layers", "-l", help="Comma-separated names of layer IDs to print"
+    ),
     dimensions: bool = Opt(False, "--dimensions", "-DIM", help="Show dimensions"),
     parameters: bool = Opt(False, "--parameters", "-PAR", help="Show parameters"),
     gradients: bool = Opt(False, "--gradients", "-GRAD", help="Show gradients"),
     attributes: bool = Opt(False, "--attributes", "-ATTR", help="Show attributes"),
     P0: bool = Opt(False, "--print-step0", "-P0", help="Print model before training"),
-    P1: bool = Opt(False, "--print-step1", "-P1", help="Print model after initialization"),
+    P1: bool = Opt(
+        False, "--print-step1", "-P1", help="Print model after initialization"
+    ),
     P2: bool = Opt(False, "--print-step2", "-P2", help="Print model after training"),
     P3: bool = Opt(False, "--print-step3", "-P3", help="Print final predictions"),
-    use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU")
+    use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
     # fmt: on
 ):
     """
@@ -81,7 +89,7 @@ def debug_model_cli(
     with show_validation_error(config_path):
         nlp = util.load_model_from_config(raw_config)
         config = nlp.config.interpolate()
-        T = registry.resolve(config["training"], schema=ConfigSchemaTraining)
+        T = registry.resolve(config["training"], schema=ConfigSchemaTraining)  # type: ignore[arg-type]
     seed = T["seed"]
     if seed is not None:
         msg.info(f"Fixing random seed: {seed}")
@@ -170,7 +178,7 @@ def debug_model(
         msg.divider(f"STEP 3 - prediction")
         msg.info(str(prediction))
 
-    msg.good(f"Succesfully ended analysis - model looks good.")
+    msg.good(f"Successfully ended analysis - model looks good.")
 
 
 def _sentences():

spacy/cli/download.py

@@ -1,5 +1,8 @@
+import importlib.util
+import shutil
 import sys
 from typing import Optional, Sequence
+from urllib.parse import urljoin
 
 import requests
 import typer
@@ -26,8 +29,16 @@ def download_cli(
     # fmt: off
     ctx: typer.Context,
     model: str = Arg(..., help="Name of pipeline package to download"),
-    direct: bool = Opt(False, "--direct", "-d", "-D", help="Force direct download of name + version"),
-    sdist: bool = Opt(False, "--sdist", "-S", help="Download sdist (.tar.gz) archive instead of pre-built binary wheel"),
+    direct: bool = Opt(
+        False, "--direct", "-d", "-D", help="Force direct download of name + version"
+    ),
+    sdist: bool = Opt(
+        False,
+        "--sdist",
+        "-S",
+        help="Download sdist (.tar.gz) archive instead of pre-built binary wheel",
+    ),
+    url: str = Opt(None, "--url", "-U", help="Download from given url"),
     # fmt: on
 ):
     """
@@ -40,13 +51,14 @@ def download_cli(
     DOCS: https://spacy.io/api/cli#download
     AVAILABLE PACKAGES: https://spacy.io/models
     """
-    download(model, direct, sdist, *ctx.args)
+    download(model, direct, sdist, url, *ctx.args)
 
 
 def download(
     model: str,
     direct: bool = False,
     sdist: bool = False,
+    custom_url: Optional[str] = None,
     *pip_args,
 ) -> None:
     if (
@@ -63,6 +75,13 @@ def download(
         )
         pip_args = pip_args + ("--no-deps",)
     if direct:
+        # Reject model names with '/', in order to prevent shenanigans.
+        if "/" in model:
+            msg.fail(
+                title="Model download rejected",
+                text=f"Cannot download model '{model}'. Models are expected to be file names, not URLs or fragments",
+                exits=True,
+            )
         components = model.split("-")
         model_name = "".join(components[:-1])
         version = components[-1]
@@ -79,7 +98,7 @@ def download(
 
     filename = get_model_filename(model_name, version, sdist)
 
-    download_model(filename, pip_args)
+    download_model(filename, pip_args, custom_url)
     msg.good(
         "Download and installation successful",
         f"You can now load the package via spacy.load('{model_name}')",
@@ -151,9 +170,34 @@ def get_latest_version(model: str) -> str:
 
 
 def download_model(
-    filename: str, user_pip_args: Optional[Sequence[str]] = None
+    filename: str,
+    user_pip_args: Optional[Sequence[str]] = None,
+    custom_url: Optional[str] = None,
 ) -> None:
-    download_url = about.__download_url__ + "/" + filename
+    # Construct the download URL carefully. We need to make sure we don't
+    # allow relative paths or other shenanigans to trick us into download
+    # from outside our own repo.
+    base_url = custom_url if custom_url else about.__download_url__
+    # urljoin requires that the path ends with /, or the last path part will be dropped
+    if not base_url.endswith("/"):
+        base_url = about.__download_url__ + "/"
+    download_url = urljoin(base_url, filename)
+    if not download_url.startswith(about.__download_url__):
+        raise ValueError(f"Download from {filename} rejected. Was it a relative path?")
     pip_args = list(user_pip_args) if user_pip_args is not None else []
-    cmd = [sys.executable, "-m", "pip", "install"] + pip_args + [download_url]
+    cmd = _get_pip_install_cmd() + pip_args + [download_url]
     run_command(cmd)
+
+
+def _get_pip_install_cmd() -> list:
+    if importlib.util.find_spec("pip") is not None:
+        return [sys.executable, "-m", "pip", "install"]
+    elif shutil.which("uv"):
+        return ["uv", "pip", "install"]
+    else:
+        msg.fail(
+            "No package installer found",
+            "spaCy requires either pip or uv to download models. "
+            "Please install one of them and try again.",
+            exits=1,
+        )

spacy/cli/evaluate.py

@@ -1,13 +1,12 @@
 import re
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional
 
 import srsly
 from thinc.api import fix_random_seed
 from wasabi import Printer
 
 from .. import displacy, util
-from ..scorer import Scorer
 from ..tokens import Doc
 from ..training import Corpus
 from ._util import Arg, Opt, app, benchmark_cli, import_code, setup_gpu
@@ -20,15 +19,42 @@ from ._util import Arg, Opt, app, benchmark_cli, import_code, setup_gpu
 def evaluate_cli(
     # fmt: off
     model: str = Arg(..., help="Model name or path"),
-    data_path: Path = Arg(..., help="Location of binary evaluation data in .spacy format", exists=True),
-    output: Optional[Path] = Opt(None, "--output", "-o", help="Output JSON file for metrics", dir_okay=False),
-    code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
+    data_path: Path = Arg(
+        ..., help="Location of binary evaluation data in .spacy format", exists=True
+    ),
+    output: Optional[Path] = Opt(
+        None, "--output", "-o", help="Output JSON file for metrics", dir_okay=False
+    ),
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
     use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
-    gold_preproc: bool = Opt(False, "--gold-preproc", "-G", help="Use gold preprocessing"),
-    displacy_path: Optional[Path] = Opt(None, "--displacy-path", "-dp", help="Directory to output rendered parses as HTML", exists=True, file_okay=False),
-    displacy_limit: int = Opt(25, "--displacy-limit", "-dl", help="Limit of parses to render as HTML"),
-    per_component: bool = Opt(False, "--per-component", "-P", help="Return scores per component, only applicable when an output JSON file is specified."),
-    spans_key: str = Opt("sc", "--spans-key", "-sk", help="Spans key to use when evaluating Doc.spans"),
+    gold_preproc: bool = Opt(
+        False, "--gold-preproc", "-G", help="Use gold preprocessing"
+    ),
+    displacy_path: Optional[Path] = Opt(
+        None,
+        "--displacy-path",
+        "-dp",
+        help="Directory to output rendered parses as HTML",
+        exists=True,
+        file_okay=False,
+    ),
+    displacy_limit: int = Opt(
+        25, "--displacy-limit", "-dl", help="Limit of parses to render as HTML"
+    ),
+    per_component: bool = Opt(
+        False,
+        "--per-component",
+        "-P",
+        help="Return scores per component, only applicable when an output JSON file is specified.",
+    ),
+    spans_key: str = Opt(
+        "sc", "--spans-key", "-sk", help="Spans key to use when evaluating Doc.spans"
+    ),
     # fmt: on
 ):
     """
@@ -123,7 +149,7 @@ def evaluate(
                     if key == "speed":
                         results[metric] = f"{scores[key]:.0f}"
                     else:
-                        results[metric] = f"{scores[key]*100:.2f}"
+                        results[metric] = f"{scores[key] * 100:.2f}"
                 else:
                     results[metric] = "-"
                 data[re.sub(r"[\s/]", "_", key.lower())] = scores[key]

spacy/cli/find_function.py

@@ -11,7 +11,9 @@ from ._util import Arg, Opt, app
 def find_function_cli(
     # fmt: off
     func_name: str = Arg(..., help="Name of the registered function."),
-    registry_name: Optional[str] = Opt(None, "--registry", "-r", help="Name of the catalogue registry."),
+    registry_name: Optional[str] = Opt(
+        None, "--registry", "-r", help="Name of the catalogue registry."
+    ),
     # fmt: on
 ):
     """

spacy/cli/find_threshold.py

@@ -27,19 +27,43 @@ _DEFAULTS = {
 def find_threshold_cli(
     # fmt: off
     model: str = Arg(..., help="Model name or path"),
-    data_path: Path = Arg(..., help="Location of binary evaluation data in .spacy format", exists=True),
+    data_path: Path = Arg(
+        ..., help="Location of binary evaluation data in .spacy format", exists=True
+    ),
     pipe_name: str = Arg(..., help="Name of pipe to examine thresholds for"),
-    threshold_key: str = Arg(..., help="Key of threshold attribute in component's configuration"),
+    threshold_key: str = Arg(
+        ..., help="Key of threshold attribute in component's configuration"
+    ),
     scores_key: str = Arg(..., help="Metric to optimize"),
-    n_trials: int = Opt(_DEFAULTS["n_trials"], "--n_trials", "-n", help="Number of trials to determine optimal thresholds"),
-    code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
-    use_gpu: int = Opt(_DEFAULTS["use_gpu"], "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
-    gold_preproc: bool = Opt(_DEFAULTS["gold_preproc"], "--gold-preproc", "-G", help="Use gold preprocessing"),
-    verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
+    n_trials: int = Opt(
+        _DEFAULTS["n_trials"],
+        "--n_trials",
+        "-n",
+        help="Number of trials to determine optimal thresholds",
+    ),
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
+    use_gpu: int = Opt(
+        _DEFAULTS["use_gpu"], "--gpu-id", "-g", help="GPU ID or -1 for CPU"
+    ),
+    gold_preproc: bool = Opt(
+        _DEFAULTS["gold_preproc"], "--gold-preproc", "-G", help="Use gold preprocessing"
+    ),
+    verbose: bool = Opt(
+        False,
+        "--verbose",
+        "-V",
+        "-VV",
+        help="Display more information for debugging purposes",
+    ),
     # fmt: on
 ):
     """
-    Runs prediction trials for a trained model with varying tresholds to maximize
+    Runs prediction trials for a trained model with varying thresholds to maximize
     the specified metric. The search space for the threshold is traversed linearly
     from 0 to 1 in `n_trials` steps. Results are displayed in a table on `stdout`
     (the corresponding API call to `spacy.cli.find_threshold.find_threshold()`
@@ -81,7 +105,7 @@ def find_threshold(
     silent: bool = True,
 ) -> Tuple[float, float, Dict[float, float]]:
     """
-    Runs prediction trials for models with varying tresholds to maximize the specified metric.
+    Runs prediction trials for models with varying thresholds to maximize the specified metric.
     model (Union[str, Path]): Pipeline to evaluate. Can be a package or a path to a data directory.
     data_path (Path): Path to file with DocBin with docs to use for threshold search.
     pipe_name (str): Name of pipe to examine thresholds for.
@@ -157,9 +181,11 @@ def find_threshold(
                 exits=1,
             )
         return {
-            keys[0]: filter_config(config[keys[0]], keys[1:], full_key)
-            if len(keys) > 1
-            else config[keys[0]]
+            keys[0]: (
+                filter_config(config[keys[0]], keys[1:], full_key)
+                if len(keys) > 1
+                else config[keys[0]]
+            )
         }
 
     # Evaluate with varying threshold values.
@@ -181,10 +207,10 @@ def find_threshold(
             ),
         )
         if hasattr(pipe, "cfg"):
-            setattr(
-                nlp.get_pipe(pipe_name),
-                "cfg",
-                set_nested_item(getattr(pipe, "cfg"), config_keys, threshold),
+            nlp.get_pipe(pipe_name).cfg = set_nested_item(  # type: ignore[attr-defined]
+                pipe.cfg,
+                config_keys,
+                threshold,  # type: ignore[attr-defined]
             )
 
         eval_scores = nlp.evaluate(dev_dataset)
@@ -216,12 +242,14 @@ def find_threshold(
     if len(set(scores.values())) == 1:
         wasabi.msg.warn(
             title="All scores are identical. Verify that all settings are correct.",
-            text=""
-            if (
-                not isinstance(pipe, MultiLabel_TextCategorizer)
-                or scores_key in ("cats_macro_f", "cats_micro_f")
-            )
-            else "Use `cats_macro_f` or `cats_micro_f` when optimizing the threshold for `textcat_multilabel`.",
+            text=(
+                ""
+                if (
+                    not isinstance(pipe, MultiLabel_TextCategorizer)
+                    or scores_key in ("cats_macro_f", "cats_micro_f")
+                )
+                else "Use `cats_macro_f` or `cats_micro_f` when optimizing the threshold for `textcat_multilabel`."
+            ),
         )
 
     else:

spacy/cli/info.py

@@ -16,10 +16,24 @@ from .download import get_latest_version, get_model_filename
 def info_cli(
     # fmt: off
     model: Optional[str] = Arg(None, help="Optional loadable spaCy pipeline"),
-    markdown: bool = Opt(False, "--markdown", "-md", help="Generate Markdown for GitHub issues"),
-    silent: bool = Opt(False, "--silent", "-s", "-S", help="Don't print anything (just return)"),
-    exclude: str = Opt("labels", "--exclude", "-e", help="Comma-separated keys to exclude from the print-out"),
-    url: bool = Opt(False, "--url", "-u", help="Print the URL to download the most recent compatible version of the pipeline"),
+    markdown: bool = Opt(
+        False, "--markdown", "-md", help="Generate Markdown for GitHub issues"
+    ),
+    silent: bool = Opt(
+        False, "--silent", "-s", "-S", help="Don't print anything (just return)"
+    ),
+    exclude: str = Opt(
+        "labels",
+        "--exclude",
+        "-e",
+        help="Comma-separated keys to exclude from the print-out",
+    ),
+    url: bool = Opt(
+        False,
+        "--url",
+        "-u",
+        help="Print the URL to download the most recent compatible version of the pipeline",
+    ),
     # fmt: on
 ):
     """

spacy/cli/init_config.py

@@ -49,13 +49,44 @@ class InitValues:
 @init_cli.command("config")
 def init_config_cli(
     # fmt: off
-    output_file: Path = Arg(..., help="File to save the config to or - for stdout (will only output config and no additional logging info)", allow_dash=True),
-    lang: str = Opt(InitValues.lang, "--lang", "-l", help="Two-letter code of the language to use"),
-    pipeline: str = Opt(",".join(InitValues.pipeline), "--pipeline", "-p", help="Comma-separated names of trainable pipeline components to include (without 'tok2vec' or 'transformer')"),
-    optimize: Optimizations = Opt(InitValues.optimize, "--optimize", "-o", help="Whether to optimize for efficiency (faster inference, smaller model, lower memory consumption) or higher accuracy (potentially larger and slower model). This will impact the choice of architecture, pretrained weights and related hyperparameters."),
-    gpu: bool = Opt(InitValues.gpu, "--gpu", "-G", help="Whether the model can run on GPU. This will impact the choice of architecture, pretrained weights and related hyperparameters."),
-    pretraining: bool = Opt(InitValues.pretraining, "--pretraining", "-pt", help="Include config for pretraining (with 'spacy pretrain')"),
-    force_overwrite: bool = Opt(InitValues.force_overwrite, "--force", "-F", help="Force overwriting the output file"),
+    output_file: Path = Arg(
+        ...,
+        help="File to save the config to or - for stdout (will only output config and no additional logging info)",
+        allow_dash=True,
+    ),
+    lang: str = Opt(
+        InitValues.lang, "--lang", "-l", help="Two-letter code of the language to use"
+    ),
+    pipeline: str = Opt(
+        ",".join(InitValues.pipeline),
+        "--pipeline",
+        "-p",
+        help="Comma-separated names of trainable pipeline components to include (without 'tok2vec' or 'transformer')",
+    ),
+    optimize: Optimizations = Opt(
+        InitValues.optimize,
+        "--optimize",
+        "-o",
+        help="Whether to optimize for efficiency (faster inference, smaller model, lower memory consumption) or higher accuracy (potentially larger and slower model). This will impact the choice of architecture, pretrained weights and related hyperparameters.",
+    ),
+    gpu: bool = Opt(
+        InitValues.gpu,
+        "--gpu",
+        "-G",
+        help="Whether the model can run on GPU. This will impact the choice of architecture, pretrained weights and related hyperparameters.",
+    ),
+    pretraining: bool = Opt(
+        InitValues.pretraining,
+        "--pretraining",
+        "-pt",
+        help="Include config for pretraining (with 'spacy pretrain')",
+    ),
+    force_overwrite: bool = Opt(
+        InitValues.force_overwrite,
+        "--force",
+        "-F",
+        help="Force overwriting the output file",
+    ),
     # fmt: on
 ):
     """
@@ -88,11 +119,28 @@ def init_config_cli(
 @init_cli.command("fill-config")
 def init_fill_config_cli(
     # fmt: off
-    base_path: Path = Arg(..., help="Path to base config to fill", exists=True, dir_okay=False),
-    output_file: Path = Arg("-", help="Path to output .cfg file (or - for stdout)", allow_dash=True),
-    pretraining: bool = Opt(False, "--pretraining", "-pt", help="Include config for pretraining (with 'spacy pretrain')"),
-    diff: bool = Opt(False, "--diff", "-D", help="Print a visual diff highlighting the changes"),
-    code_path: Optional[Path] = Opt(None, "--code-path", "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
+    base_path: Path = Arg(
+        ..., help="Path to base config to fill", exists=True, dir_okay=False
+    ),
+    output_file: Path = Arg(
+        "-", help="Path to output .cfg file (or - for stdout)", allow_dash=True
+    ),
+    pretraining: bool = Opt(
+        False,
+        "--pretraining",
+        "-pt",
+        help="Include config for pretraining (with 'spacy pretrain')",
+    ),
+    diff: bool = Opt(
+        False, "--diff", "-D", help="Print a visual diff highlighting the changes"
+    ),
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code-path",
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
     # fmt: on
 ):
     """
@@ -168,7 +216,7 @@ def init_config(
     # Filter out duplicates since tok2vec and transformer are added by template
     pipeline = [pipe for pipe in pipeline if pipe not in ("tok2vec", "transformer")]
     defaults = RECOMMENDATIONS["__default__"]
-    reco = RecommendationSchema(**RECOMMENDATIONS.get(lang, defaults)).dict()
+    reco = RecommendationSchema(**RECOMMENDATIONS.get(lang, defaults)).model_dump()
     variables = {
         "lang": lang,
         "components": pipeline,
@@ -195,9 +243,11 @@ def init_config(
         "Pipeline": ", ".join(pipeline),
         "Optimize for": optimize,
         "Hardware": variables["hardware"].upper(),
-        "Transformer": template_vars.transformer.get("name")  # type: ignore[attr-defined]
-        if template_vars.use_transformer  # type: ignore[attr-defined]
-        else None,
+        "Transformer": (
+            template_vars.transformer.get("name")  # type: ignore[attr-defined]
+            if template_vars.use_transformer  # type: ignore[attr-defined]
+            else None
+        ),
     }
     msg.info("Generated config template specific for your use case")
     for label, value in use_case.items():

spacy/cli/init_pipeline.py

@@ -26,13 +26,42 @@ def init_vectors_cli(
     lang: str = Arg(..., help="The language of the nlp object to create"),
     vectors_loc: Path = Arg(..., help="Vectors file in Word2Vec format", exists=True),
     output_dir: Path = Arg(..., help="Pipeline output directory"),
-    prune: int = Opt(-1, "--prune", "-p", help="Optional number of vectors to prune to"),
-    truncate: int = Opt(0, "--truncate", "-t", help="Optional number of vectors to truncate to when reading in vectors file"),
+    prune: int = Opt(
+        -1, "--prune", "-p", help="Optional number of vectors to prune to"
+    ),
+    truncate: int = Opt(
+        0,
+        "--truncate",
+        "-t",
+        help="Optional number of vectors to truncate to when reading in vectors file",
+    ),
     mode: str = Opt("default", "--mode", "-m", help="Vectors mode: default or floret"),
-    name: Optional[str] = Opt(None, "--name", "-n", help="Optional name for the word vectors, e.g. en_core_web_lg.vectors"),
-    verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
-    jsonl_loc: Optional[Path] = Opt(None, "--lexemes-jsonl", "-j", help="Location of JSONL-formatted attributes file", hidden=True),
-    attr: str = Opt("ORTH", "--attr", "-a", help="Optional token attribute to use for vectors, e.g. LOWER or NORM"),
+    name: Optional[str] = Opt(
+        None,
+        "--name",
+        "-n",
+        help="Optional name for the word vectors, e.g. en_core_web_lg.vectors",
+    ),
+    verbose: bool = Opt(
+        False,
+        "--verbose",
+        "-V",
+        "-VV",
+        help="Display more information for debugging purposes",
+    ),
+    jsonl_loc: Optional[Path] = Opt(
+        None,
+        "--lexemes-jsonl",
+        "-j",
+        help="Location of JSONL-formatted attributes file",
+        hidden=True,
+    ),
+    attr: str = Opt(
+        "ORTH",
+        "--attr",
+        "-a",
+        help="Optional token attribute to use for vectors, e.g. LOWER or NORM",
+    ),
     # fmt: on
 ):
     """Convert word vectors for use with spaCy. Will export an nlp object that
@@ -81,11 +110,24 @@ def update_lexemes(nlp: Language, jsonl_loc: Path) -> None:
 def init_pipeline_cli(
     # fmt: off
     ctx: typer.Context,  # This is only used to read additional arguments
-    config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
+    config_path: Path = Arg(
+        ..., help="Path to config file", exists=True, allow_dash=True
+    ),
     output_path: Path = Arg(..., help="Output directory for the prepared data"),
-    code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
-    verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
-    use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU")
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
+    verbose: bool = Opt(
+        False,
+        "--verbose",
+        "-V",
+        "-VV",
+        help="Display more information for debugging purposes",
+    ),
+    use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
     # fmt: on
 ):
     if verbose:
@@ -108,11 +150,24 @@ def init_pipeline_cli(
 def init_labels_cli(
     # fmt: off
     ctx: typer.Context,  # This is only used to read additional arguments
-    config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
+    config_path: Path = Arg(
+        ..., help="Path to config file", exists=True, allow_dash=True
+    ),
     output_path: Path = Arg(..., help="Output directory for the labels"),
-    code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
-    verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
-    use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU")
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
+    verbose: bool = Opt(
+        False,
+        "--verbose",
+        "-V",
+        "-VV",
+        help="Display more information for debugging purposes",
+    ),
+    use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
     # fmt: on
 ):
     """Generate JSON files for the labels in the data. This helps speed up the

spacy/cli/package.py

@@ -21,15 +21,56 @@ from ._util import SDIST_SUFFIX, WHEEL_SUFFIX, Arg, Opt, app, string_to_list
 @app.command("package")
 def package_cli(
     # fmt: off
-    input_dir: Path = Arg(..., help="Directory with pipeline data", exists=True, file_okay=False),
-    output_dir: Path = Arg(..., help="Output parent directory", exists=True, file_okay=False),
-    code_paths: str = Opt("", "--code", "-c", help="Comma-separated paths to Python file with additional code (registered functions) to be included in the package"),
-    meta_path: Optional[Path] = Opt(None, "--meta-path", "--meta", "-m", help="Path to meta.json", exists=True, dir_okay=False),
-    create_meta: bool = Opt(False, "--create-meta", "-C", help="Create meta.json, even if one exists"),
-    name: Optional[str] = Opt(None, "--name", "-n", help="Package name to override meta"),
-    version: Optional[str] = Opt(None, "--version", "-v", help="Package version to override meta"),
-    build: str = Opt("sdist", "--build", "-b", help="Comma-separated formats to build: sdist and/or wheel, or none."),
-    force: bool = Opt(False, "--force", "-f", "-F", help="Force overwriting existing data in output directory"),
+    input_dir: Path = Arg(
+        ..., help="Directory with pipeline data", exists=True, file_okay=False
+    ),
+    output_dir: Path = Arg(
+        ..., help="Output parent directory", exists=True, file_okay=False
+    ),
+    code_paths: str = Opt(
+        "",
+        "--code",
+        "-c",
+        help="Comma-separated paths to Python file with additional code (registered functions) to be included in the package",
+    ),
+    meta_path: Optional[Path] = Opt(
+        None,
+        "--meta-path",
+        "--meta",
+        "-m",
+        help="Path to meta.json",
+        exists=True,
+        dir_okay=False,
+    ),
+    create_meta: bool = Opt(
+        False, "--create-meta", "-C", help="Create meta.json, even if one exists"
+    ),
+    name: Optional[str] = Opt(
+        None, "--name", "-n", help="Package name to override meta"
+    ),
+    version: Optional[str] = Opt(
+        None, "--version", "-v", help="Package version to override meta"
+    ),
+    build: str = Opt(
+        "sdist",
+        "--build",
+        "-b",
+        help="Comma-separated formats to build: sdist and/or wheel, or none.",
+    ),
+    force: bool = Opt(
+        False,
+        "--force",
+        "-f",
+        "-F",
+        help="Force overwriting existing data in output directory",
+    ),
+    require_parent: bool = Opt(
+        True,
+        "--require-parent/--no-require-parent",
+        "-R",
+        "-R",
+        help="Include the parent package (e.g. spacy) in the requirements",
+    ),
     # fmt: on
 ):
     """
@@ -60,6 +101,7 @@ def package_cli(
         create_sdist=create_sdist,
         create_wheel=create_wheel,
         force=force,
+        require_parent=require_parent,
         silent=False,
     )
 
@@ -74,6 +116,7 @@ def package(
     create_meta: bool = False,
     create_sdist: bool = True,
     create_wheel: bool = False,
+    require_parent: bool = False,
     force: bool = False,
     silent: bool = True,
 ) -> None:
@@ -113,7 +156,7 @@ def package(
     if not meta_path.exists() or not meta_path.is_file():
         msg.fail("Can't load pipeline meta.json", meta_path, exits=1)
     meta = srsly.read_json(meta_path)
-    meta = get_meta(input_dir, meta)
+    meta = get_meta(input_dir, meta, require_parent=require_parent)
     if meta["requirements"]:
         msg.good(
             f"Including {len(meta['requirements'])} package requirement(s) from "
@@ -186,6 +229,7 @@ def package(
         imports.append(code_path.stem)
         shutil.copy(str(code_path), str(package_path))
     create_file(main_path / "meta.json", srsly.json_dumps(meta, indent=2))
+
     create_file(main_path / "setup.py", TEMPLATE_SETUP)
     create_file(main_path / "MANIFEST.in", TEMPLATE_MANIFEST)
     init_py = TEMPLATE_INIT.format(
@@ -302,6 +346,8 @@ def get_third_party_dependencies(
                 modules.add(func_info["module"].split(".")[0])  # type: ignore[union-attr]
     dependencies = []
     for module_name in modules:
+        if module_name == about.__title__:
+            continue
         if module_name in distributions:
             dist = distributions.get(module_name)
             if dist:
@@ -332,7 +378,9 @@ def create_file(file_path: Path, contents: str) -> None:
 
 
 def get_meta(
-    model_path: Union[str, Path], existing_meta: Dict[str, Any]
+    model_path: Union[str, Path],
+    existing_meta: Dict[str, Any],
+    require_parent: bool = False,
 ) -> Dict[str, Any]:
     meta: Dict[str, Any] = {
         "lang": "en",
@@ -361,6 +409,8 @@ def get_meta(
     existing_reqs = [util.split_requirement(req)[0] for req in meta["requirements"]]
     reqs = get_third_party_dependencies(nlp.config, exclude=existing_reqs)
     meta["requirements"].extend(reqs)
+    if require_parent and about.__title__ not in meta["requirements"]:
+        meta["requirements"].append(about.__title__ + meta["spacy_version"])
     return meta
 
 
@@ -400,7 +450,7 @@ def generate_readme(meta: Dict[str, Any]) -> str:
     pipeline = ", ".join([md.code(p) for p in meta.get("pipeline", [])])
     components = ", ".join([md.code(p) for p in meta.get("components", [])])
     vecs = meta.get("vectors", {})
-    vectors = f"{vecs.get('keys', 0)} keys, {vecs.get('vectors', 0)} unique vectors ({ vecs.get('width', 0)} dimensions)"
+    vectors = f"{vecs.get('keys', 0)} keys, {vecs.get('vectors', 0)} unique vectors ({vecs.get('width', 0)} dimensions)"
     author = meta.get("author") or "n/a"
     notes = meta.get("notes", "")
     license_name = meta.get("license")
@@ -459,7 +509,7 @@ def _format_accuracy(data: Dict[str, Any], exclude: List[str] = ["speed"]) -> st
     md = MarkdownRenderer()
     scalars = [(k, v) for k, v in data.items() if isinstance(v, (int, float))]
     scores = [
-        (md.code(acc.upper()), f"{score*100:.2f}")
+        (md.code(acc.upper()), f"{score * 100:.2f}")
         for acc, score in scalars
         if acc not in exclude
     ]
@@ -478,9 +528,7 @@ def _format_label_scheme(data: Dict[str, Any]) -> str:
         if not labels:
             continue
         col1 = md.bold(md.code(pipe))
-        col2 = ", ".join(
-            [md.code(str(label).replace("|", "\\|")) for label in labels]
-        )  # noqa: W605
+        col2 = ", ".join([md.code(str(label).replace("|", "\\|")) for label in labels])  # noqa: W605
         label_data.append((col1, col2))
         n_labels += len(labels)
         n_pipes += 1
@@ -535,8 +583,11 @@ def list_files(data_dir):
 
 
 def list_requirements(meta):
-    parent_package = meta.get('parent_package', 'spacy')
-    requirements = [parent_package + meta['spacy_version']]
+    # Up to version 3.7, we included the parent package
+    # in requirements by default. This behaviour is removed
+    # in 3.8, with a setting to include the parent package in
+    # the requirements list in the meta if desired.
+    requirements = []
     if 'setup_requires' in meta:
         requirements += meta['setup_requires']
     if 'requirements' in meta:

spacy/cli/pretrain.py

@@ -25,13 +25,32 @@ from ._util import (
 def pretrain_cli(
     # fmt: off
     ctx: typer.Context,  # This is only used to read additional arguments
-    config_path: Path = Arg(..., help="Path to config file", exists=True, dir_okay=False, allow_dash=True),
+    config_path: Path = Arg(
+        ..., help="Path to config file", exists=True, dir_okay=False, allow_dash=True
+    ),
     output_dir: Path = Arg(..., help="Directory to write weights to on each epoch"),
-    code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
-    resume_path: Optional[Path] = Opt(None, "--resume-path", "-r", help="Path to pretrained weights from which to resume pretraining"),
-    epoch_resume: Optional[int] = Opt(None, "--epoch-resume", "-er", help="The epoch to resume counting from when using --resume-path. Prevents unintended overwriting of existing weight files."),
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
+    resume_path: Optional[Path] = Opt(
+        None,
+        "--resume-path",
+        "-r",
+        help="Path to pretrained weights from which to resume pretraining",
+    ),
+    epoch_resume: Optional[int] = Opt(
+        None,
+        "--epoch-resume",
+        "-er",
+        help="The epoch to resume counting from when using --resume-path. Prevents unintended overwriting of existing weight files.",
+    ),
     use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
-    skip_last: bool = Opt(False, "--skip-last", "-L", help="Skip saving model-last.bin"),
+    skip_last: bool = Opt(
+        False, "--skip-last", "-L", help="Skip saving model-last.bin"
+    ),
     # fmt: on
 ):
     """

spacy/cli/profile.py

@@ -21,8 +21,15 @@ def profile_cli(
     # fmt: off
     ctx: typer.Context,  # This is only used to read current calling context
     model: str = Arg(..., help="Trained pipeline to load"),
-    inputs: Optional[Path] = Arg(None, help="Location of input file. '-' for stdin.", exists=True, allow_dash=True),
-    n_texts: int = Opt(10000, "--n-texts", "-n", help="Maximum number of texts to use if available"),
+    inputs: Optional[Path] = Arg(
+        None,
+        help="Location of input file. '-' for stdin.",
+        exists=True,
+        allow_dash=True,
+    ),
+    n_texts: int = Opt(
+        10000, "--n-texts", "-n", help="Maximum number of texts to use if available"
+    ),
     # fmt: on
 ):
     """
@@ -59,7 +66,7 @@ def profile(model: str, inputs: Optional[Path] = None, n_texts: int = 10000) ->
 
         with msg.loading("Loading IMDB dataset via ml_datasets..."):
             imdb_train, _ = ml_datasets.imdb(train_limit=n_texts, dev_limit=0)
-            texts, _ = zip(*imdb_train)
+            texts = [text for text, _ in imdb_train]
         msg.info(f"Loaded IMDB dataset and using {n_texts} examples")
     with msg.loading(f"Loading pipeline '{model}'..."):
         nlp = load_model(model)

spacy/cli/train.py

@@ -26,11 +26,30 @@ from ._util import (
 def train_cli(
     # fmt: off
     ctx: typer.Context,  # This is only used to read additional arguments
-    config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
-    output_path: Optional[Path] = Opt(None, "--output", "--output-path", "-o", help="Output directory to store trained pipeline in"),
-    code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
-    verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
-    use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU")
+    config_path: Path = Arg(
+        ..., help="Path to config file", exists=True, allow_dash=True
+    ),
+    output_path: Optional[Path] = Opt(
+        None,
+        "--output",
+        "--output-path",
+        "-o",
+        help="Output directory to store trained pipeline in",
+    ),
+    code_path: Optional[Path] = Opt(
+        None,
+        "--code",
+        "-c",
+        help="Path to Python file with additional code (registered functions) to be imported",
+    ),
+    verbose: bool = Opt(
+        False,
+        "--verbose",
+        "-V",
+        "-VV",
+        help="Display more information for debugging purposes",
+    ),
+    use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
     # fmt: on
 ):
     """

spacy/compat.py

@@ -1,4 +1,5 @@
 """Helpers for Python and platform compatibility."""
+
 import sys
 
 from thinc.util import copy_array
@@ -34,7 +35,9 @@ else:
 try:  # Python 3.8+
     import importlib.metadata as importlib_metadata
 except ImportError:
-    from catalogue import _importlib_metadata as importlib_metadata  # type: ignore[no-redef]    # noqa: F401
+    from catalogue import (  # type: ignore[no-redef]
+        _importlib_metadata as importlib_metadata,  # noqa: F401
+    )
 
 from thinc.api import Optimizer  # noqa: F401
 

spacy/displacy/__init__.py

@@ -4,6 +4,7 @@ spaCy's built in visualization suite for dependencies and named entities.
 DOCS: https://spacy.io/api/top-level#displacy
 USAGE: https://spacy.io/usage/visualizers
 """
+
 import warnings
 from typing import Any, Callable, Dict, Iterable, Optional, Union
 
@@ -66,7 +67,7 @@ def render(
     if jupyter or (jupyter is None and is_in_jupyter()):
         # return HTML rendered by IPython display()
         # See #4840 for details on span wrapper to disable mathjax
-        from IPython.core.display import HTML, display
+        from IPython.display import HTML, display
 
         return display(HTML('<span class="tex2jax_ignore">{}</span>'.format(html)))
     return html

spacy/displacy/render.py

@@ -388,7 +388,7 @@ class DependencyRenderer:
             lang=self.lang,
         )
 
-    def render_word(self, text: str, tag: str, lemma: str, i: int) -> str:
+    def render_word(self, text: str, tag: str, lemma: Optional[str], i: int) -> str:
         """Render individual word.
 
         text (str): Word text.

spacy/errors.py

@@ -220,6 +220,7 @@ class Warnings(metaclass=ErrorsWithCodes):
             "key attribute for vectors, configure it through Vectors(attr=) or "
             "'spacy init vectors --attr'")
     W126 = ("These keys are unsupported: {unsupported}")
+    W127 = ("Not all `Language.pipe` worker processes completed successfully")
 
 
 class Errors(metaclass=ErrorsWithCodes):

spacy/lang/am/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "አፕል የዩኬን ጅምር ድርጅት በ 1 ቢሊዮን ዶላር ለመግዛት አስቧል።",
     "የራስ ገዝ መኪኖች የኢንሹራንስ ኃላፊነትን ወደ አምራቾች ያዛውራሉ",

spacy/lang/am/lex_attrs.py

@@ -60,7 +60,7 @@ _ordinal_words = [
     "አስራ ስምንተኛ",
     "አስራ ዘጠነኛ",
     "ሃያኛ",
-    "ሰላሳኛ" "አርባኛ",
+    "ሰላሳኛአርባኛ",
     "አምሳኛ",
     "ስድሳኛ",
     "ሰባኛ",

spacy/lang/az/examples.py

@@ -4,7 +4,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Bu bir cümlədir.",
     "Necəsən?",

spacy/lang/bg/stop_words.py

@@ -3,6 +3,7 @@ References:
     https://github.com/Alir3z4/stop-words - Original list, serves as a base.
     https://postvai.com/books/stop-dumi.pdf - Additions to the original list in order to improve it.
 """
+
 STOP_WORDS = set(
     """
 а автентичен аз ако ала

spacy/lang/bn/examples.py

@@ -5,5 +5,4 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = ["তুই খুব ভালো", "আজ আমরা ডাক্তার দেখতে যাবো", "আমি জানি না "]

spacy/lang/bo/__init__.py

@@ -0,0 +1,16 @@
+from ...language import BaseDefaults, Language
+from .lex_attrs import LEX_ATTRS
+from .stop_words import STOP_WORDS
+
+
+class TibetanDefaults(BaseDefaults):
+    lex_attr_getters = LEX_ATTRS
+    stop_words = STOP_WORDS
+
+
+class Tibetan(Language):
+    lang = "bo"
+    Defaults = TibetanDefaults
+
+
+__all__ = ["Tibetan"]

spacy/lang/bo/examples.py

@@ -0,0 +1,15 @@
+"""
+Example sentences to test spaCy and its language models.
+
+>>> from spacy.lang.bo.examples import sentences
+>>> docs = nlp.pipe(sentences)
+"""
+
+sentences = [
+    "དོན་དུ་རྒྱ་མཚོ་བླ་མ་ཞེས་བྱ་ཞིང༌།",
+    "ཏཱ་ལའི་ཞེས་པ་ནི་སོག་སྐད་ཡིན་པ་དེ་བོད་སྐད་དུ་རྒྱ་མཚོའི་དོན་དུ་འཇུག",
+    "སོག་པོ་ཨལ་ཐན་རྒྱལ་པོས་རྒྱལ་དབང་བསོད་ནམས་རྒྱ་མཚོར་ཆེ་བསྟོད་ཀྱི་མཚན་གསོལ་བ་ཞིག་ཡིན་ཞིང༌།",
+    "རྗེས་སུ་རྒྱལ་བ་དགེ་འདུན་གྲུབ་དང༌། དགེ་འདུན་རྒྱ་མཚོ་སོ་སོར་ཡང་ཏཱ་ལའི་བླ་མའི་སྐུ་ཕྲེང་དང་པོ་དང༌།",
+    "གཉིས་པའི་མཚན་དེ་གསོལ་ཞིང༌།༸རྒྱལ་དབང་སྐུ་ཕྲེང་ལྔ་པས་དགའ་ལྡན་ཕོ་བྲང་གི་སྲིད་དབང་བཙུགས་པ་ནས་ཏཱ་ལའི་བླ་མ་ནི་བོད་ཀྱི་ཆོས་སྲིད་གཉིས་ཀྱི་དབུ་ཁྲིད་དུ་གྱུར་ཞིང་།",
+    "ད་ལྟའི་བར་ཏཱ་ལའི་བླ་མ་སྐུ་ཕྲེང་བཅུ་བཞི་བྱོན་ཡོད།",
+]

spacy/lang/bo/lex_attrs.py

@@ -0,0 +1,65 @@
+from ...attrs import LIKE_NUM
+
+# reference 1: https://en.wikipedia.org/wiki/Tibetan_numerals
+
+_num_words = [
+    "ཀླད་ཀོར་",
+    "གཅིག་",
+    "གཉིས་",
+    "གསུམ་",
+    "བཞི་",
+    "ལྔ་",
+    "དྲུག་",
+    "བདུན་",
+    "བརྒྱད་",
+    "དགུ་",
+    "བཅུ་",
+    "བཅུ་གཅིག་",
+    "བཅུ་གཉིས་",
+    "བཅུ་གསུམ་",
+    "བཅུ་བཞི་",
+    "བཅུ་ལྔ་",
+    "བཅུ་དྲུག་",
+    "བཅུ་བདུན་",
+    "བཅུ་པརྒྱད",
+    "བཅུ་དགུ་",
+    "ཉི་ཤུ་",
+    "སུམ་ཅུ",
+    "བཞི་བཅུ",
+    "ལྔ་བཅུ",
+    "དྲུག་ཅུ",
+    "བདུན་ཅུ",
+    "བརྒྱད་ཅུ",
+    "དགུ་བཅུ",
+    "བརྒྱ་",
+    "སྟོང་",
+    "ཁྲི་",
+    "ས་ཡ་",
+    "	བྱེ་བ་",
+    "དུང་ཕྱུར་",
+    "ཐེར་འབུམ་",
+    "ཐེར་འབུམ་ཆེན་པོ་",
+    "ཁྲག་ཁྲིག་",
+    "ཁྲག་ཁྲིག་ཆེན་པོ་",
+]
+
+
+def like_num(text):
+    """
+    Check if text resembles a number
+    """
+    if text.startswith(("+", "-", "±", "~")):
+        text = text[1:]
+    text = text.replace(",", "").replace(".", "")
+    if text.isdigit():
+        return True
+    if text.count("/") == 1:
+        num, denom = text.split("/")
+        if num.isdigit() and denom.isdigit():
+            return True
+    if text in _num_words:
+        return True
+    return False
+
+
+LEX_ATTRS = {LIKE_NUM: like_num}

spacy/lang/bo/stop_words.py

@@ -0,0 +1,198 @@
+# Source: https://zenodo.org/records/10148636
+
+STOP_WORDS = set(
+    """
+འི་
+།
+དུ་
+གིས་
+སོགས་
+ཏེ
+གི་
+རྣམས་
+ནི
+ཀུན་
+ཡི་
+འདི
+ཀྱི་
+སྙེད་
+པས་
+གཞན་
+ཀྱིས་
+ཡི
+ལ
+ནི་
+དང་
+སོགས
+ཅིང་
+ར
+དུ
+མི་
+སུ་
+བཅས་
+ཡོངས་
+ལས
+ཙམ་
+གྱིས་
+དེ་
+ཡང་
+མཐའ་དག་
+ཏུ་
+ཉིད་
+ས
+ཏེ་
+གྱི་
+སྤྱི
+དེ
+ཀ་
+ཡིན་
+ཞིང་
+འདི་
+རུང་
+རང་
+ཞིག་
+སྟེ
+སྟེ་
+ན་རེ
+ངམ
+ཤིང་
+དག་
+ཏོ
+རེ་
+འང་
+ཀྱང་
+ལགས་པ
+ཚུ
+དོ
+ཡིན་པ
+རེ
+ན་རེ་
+ཨེ་
+ཚང་མ
+ཐམས་ཅད་
+དམ་
+འོ་
+ཅིག་
+གྱིན་
+ཡིན
+ན
+ཁོ་ན་
+འམ་
+ཀྱིན་
+ལོ
+ཀྱིས
+བས་
+ལགས་
+ཤིག
+གིས
+ཀི་
+སྣ་ཚོགས་
+རྣམས
+སྙེད་པ
+ཡིས་
+གྱི
+གི
+བམ་
+ཤིག་
+རེ་རེ་
+ནམ
+མིན་
+ནམ་
+ངམ་
+རུ་
+འགའ་
+ཀུན
+ཤས་
+ཏུ
+ཡིས
+གིན་
+གམ་
+འོ
+ཡིན་པ་
+མིན
+ལགས
+གྱིས
+ཅང་
+འགའ
+སམ་
+ཞིག
+འང
+ལས་ཆེ་
+འཕྲལ་
+བར་
+རུ
+དང
+ཡ
+འག
+སམ
+ཀ
+ཅུང་ཟད་
+ཅིག
+ཉིད
+དུ་མ
+མ
+ཡིན་བ
+འམ
+མམ
+དམ
+དག
+ཁོ་ན
+ཀྱི
+ལམ
+ཕྱི་
+ནང་
+ཙམ
+ནོ་
+སོ་
+རམ་
+བོ་
+ཨང་
+ཕྱི
+ཏོ་
+ཚོ
+ལ་ལ་
+ཚོ་
+ཅིང
+མ་གི་
+གེ
+གོ
+ཡིན་ལུགས་
+རོ་
+བོ
+ལགས་པ་
+པས
+རབ་
+འི
+རམ
+བས
+གཞན
+སྙེད་པ་
+འབའ་
+མཾ་
+པོ
+ག་
+ག
+གམ
+སྤྱི་
+བམ
+མོ་
+ཙམ་པ་
+ཤ་སྟག་
+མམ་
+རེ་རེ
+སྙེད
+ཏམ་
+ངོ
+གྲང་
+ཏ་རེ
+ཏམ
+ཁ་
+ངེ་
+ཅོག་
+རིལ་
+ཉུང་ཤས་
+གིང་
+ཚ་
+ཀྱང
+""".split()
+)

spacy/lang/ca/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Apple està buscant comprar una startup del Regne Unit per mil milions de dòlars",
     "Els cotxes autònoms deleguen la responsabilitat de l'assegurança als seus fabricants",

spacy/lang/char_classes.py

@@ -277,10 +277,10 @@ _currency = (
 # These expressions contain various unicode variations, including characters
 # used in Chinese (see #1333, #1340, #1351) – unless there are cross-language
 # conflicts, spaCy's base tokenizer should handle all of those by default
-_punct = (
-    r"… …… , : ; \! \? ¿ ؟ ¡ \( \) \[ \] \{ \} < > _ # \* & 。 ？ ！ ， 、 ； ： ～ · । ، ۔ ؛ ٪"
+_punct = r"… …… , : ; \! \? ¿ ؟ ¡ \( \) \[ \] \{ \} < > _ # \* & 。 ？ ！ ， 、 ； ： ～ · । ، ۔ ؛ ٪"
+_quotes = (
+    r'\' " ” “ ` ‘ ´ ’ ‚ , „ » « 「 」 『 』 （ ） 〔 〕 【 】 《 》 〈 〉 〈 〉  ⟦ ⟧'
 )
-_quotes = r'\' " ” “ ` ‘ ´ ’ ‚ , „ » « 「 」 『 』 （ ） 〔 〕 【 】 《 》 〈 〉 〈 〉  ⟦ ⟧'
 _hyphens = "- – — -- --- —— ~"
 
 # Various symbols like dingbats, but also emoji

spacy/lang/cs/examples.py

@@ -4,7 +4,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Máma mele maso.",
     "Příliš žluťoučký kůň úpěl ďábelské ódy.",

spacy/lang/da/tokenizer_exceptions.py

@@ -2,6 +2,7 @@
 Tokenizer Exceptions.
 Source: https://forkortelse.dk/ and various others.
 """
+
 from ...symbols import NORM, ORTH
 from ...util import update_exc
 from ..tokenizer_exceptions import BASE_EXCEPTIONS

spacy/lang/de/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Die ganze Stadt ist ein Startup: Shenzhen ist das Silicon Valley für Hardware-Firmen",
     "Wie deutsche Startups die Technologie vorantreiben wollen: Künstliche Intelligenz",

spacy/lang/dsb/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Z tym stwori so wuměnjenje a zakład za dalše wobdźěłanje přez analyzu tekstoweje struktury a semantisku anotaciju a z tym tež za tu předstajenu digitalnu online-wersiju.",
     "Mi so tu jara derje spodoba.",

spacy/lang/el/tokenizer_exceptions.py

@@ -128,7 +128,6 @@ _other_exc = {
 _exc.update(_other_exc)
 
 for h in range(1, 12 + 1):
-
     for period in ["π.μ.", "πμ"]:
         _exc[f"{h}{period}"] = [
             {ORTH: f"{h}"},

spacy/lang/en/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Apple is looking at buying U.K. startup for $1 billion",
     "Autonomous cars shift insurance liability toward manufacturers",

spacy/lang/es/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Apple está buscando comprar una startup del Reino Unido por mil millones de dólares.",
     "Los coches autónomos delegan la responsabilidad del seguro en sus fabricantes.",

spacy/lang/es/lemmatizer.py

@@ -415,7 +415,10 @@ class SpanishLemmatizer(Lemmatizer):
         else:
             rule = self.select_rule("verb", features)
             verb_lemma = self.lemmatize_verb(
-                verb, features - {"PronType=Prs"}, rule, index  # type: ignore[operator]
+                verb,
+                features - {"PronType=Prs"},  # type: ignore[operator]
+                rule,
+                index,  # type: ignore[operator]
             )[0]
         pron_lemmas = []
         for pron in prons:

spacy/lang/fa/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "این یک جمله نمونه می باشد.",
     "قرار ما، امروز ساعت ۲:۳۰ بعدازظهر هست!",

spacy/lang/fa/generate_verbs_exc.py

@@ -611,8 +611,8 @@ narrative_ends = ["ه‌ام", "ه‌ای", "ه", "ه‌ایم", "ه‌اید",
 present_ends = ["م", "ی", "د", "یم", "ید", "ند"]
 
 # special case of '#هست':
-VERBS_EXC.update({conj: "هست" for conj in ["هست" + end for end in simple_ends]})
-VERBS_EXC.update({conj: "هست" for conj in ["نیست" + end for end in simple_ends]})
+VERBS_EXC.update(dict.fromkeys(["هست" + end for end in simple_ends], "هست"))
+VERBS_EXC.update(dict.fromkeys(["نیست" + end for end in simple_ends], "هست"))
 
 for verb_root in verb_roots:
     conjugations = []
@@ -648,4 +648,4 @@ for verb_root in verb_roots:
             )
         )
 
-    VERBS_EXC.update({conj: (past,) if past else present for conj in conjugations})
+    VERBS_EXC.update(dict.fromkeys(conjugations, (past,) if past else present))

spacy/lang/fi/tokenizer_exceptions.py

@@ -100,9 +100,9 @@ conj_contraction_negations = [
     ("eivat", "eivät"),
     ("eivät", "eivät"),
 ]
-for (base_lower, base_norm) in conj_contraction_bases:
+for base_lower, base_norm in conj_contraction_bases:
     for base in [base_lower, base_lower.title()]:
-        for (suffix, suffix_norm) in conj_contraction_negations:
+        for suffix, suffix_norm in conj_contraction_negations:
             _exc[base + suffix] = [
                 {ORTH: base, NORM: base_norm},
                 {ORTH: suffix, NORM: suffix_norm},

spacy/lang/fr/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Apple cherche à acheter une start-up anglaise pour 1 milliard de dollars",
     "Les voitures autonomes déplacent la responsabilité de l'assurance vers les constructeurs",

spacy/lang/ga/lemmatizer.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Tuple
+from typing import List, Tuple
 
 from ...pipeline import Lemmatizer
 from ...tokens import Token

spacy/lang/gd/__init__.py

@@ -0,0 +1,18 @@
+from typing import Optional
+
+from ...language import BaseDefaults, Language
+from .stop_words import STOP_WORDS
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
+
+
+class ScottishDefaults(BaseDefaults):
+    tokenizer_exceptions = TOKENIZER_EXCEPTIONS
+    stop_words = STOP_WORDS
+
+
+class Scottish(Language):
+    lang = "gd"
+    Defaults = ScottishDefaults
+
+
+__all__ = ["Scottish"]

spacy/lang/gd/stop_words.py

@@ -0,0 +1,386 @@
+STOP_WORDS = set(
+    """
+'ad
+'ar
+'d # iad
+'g # ag
+'ga
+'gam
+'gan
+'gar
+'gur
+'m # am
+'n # an
+'n seo
+'na
+'nad
+'nam
+'nan
+'nar
+'nuair
+'nur
+'s
+'sa
+'san
+'sann
+'se
+'sna
+a
+a'
+a'd # agad
+a'm # agam
+a-chèile
+a-seo
+a-sin
+a-siud
+a chionn
+a chionn 's
+a chèile
+a chéile
+a dh'
+a h-uile
+a seo
+ac' # aca
+aca
+aca-san
+acasan
+ach
+ag
+agad
+agad-sa
+agads'
+agadsa
+agaibh
+agaibhse
+againn
+againne
+agam
+agam-sa
+agams'
+agamsa
+agus
+aice
+aice-se
+aicese
+aig
+aig' # aige
+aige
+aige-san
+aigesan
+air
+air-san
+air neo
+airsan
+am
+an
+an seo
+an sin
+an siud
+an uair
+ann
+ann a
+ann a'
+ann a shin
+ann am
+ann an
+annad
+annam
+annam-s'
+annamsa
+anns
+anns an
+annta
+aon
+ar
+as
+asad
+asda
+asta
+b'
+bho
+bhon
+bhuaidhe # bhuaithe
+bhuainn
+bhuaipe
+bhuaithe
+bhuapa
+bhur
+brì
+bu
+c'à
+car son
+carson
+cha
+chan
+chionn
+choir
+chon
+chun
+chèile
+chéile
+chòir
+cia mheud
+ciamar
+co-dhiubh
+cuide
+cuin
+cuin'
+cuine
+cà
+cà'
+càil
+càit
+càit'
+càite
+cò
+cò mheud
+có
+d'
+da
+de
+dh'
+dha
+dhaibh
+dhaibh-san
+dhaibhsan
+dhan
+dhasan
+dhe
+dhen
+dheth
+dhi
+dhiom
+dhiot
+dhith
+dhiubh
+dhomh
+dhomh-s'
+dhomhsa
+dhu'sa # dhut-sa
+dhuibh
+dhuibhse
+dhuinn
+dhuinne
+dhuit
+dhut
+dhutsa
+dhut-sa
+dhà
+dhà-san
+dhàsan
+dhòmhsa
+diubh
+do
+docha
+don
+dà
+dè
+dè mar
+dé
+dé mar
+dòch'
+dòcha
+e
+eadar
+eatarra
+eatorra
+eile
+esan
+fa
+far
+feud
+fhad
+fheudar
+fhearr
+fhein
+fheudar
+fheàrr
+fhèin
+fhéin
+fhìn
+fo
+fodha
+fodhainn
+foipe
+fon
+fèin
+ga
+gach
+gam
+gan
+ge brith
+ged
+gu
+gu dè
+gu ruige
+gun
+gur
+gus
+i
+iad
+iadsan
+innte
+is
+ise
+le
+leam
+leam-sa
+leamsa
+leat
+leat-sa
+leatha
+leatsa
+leibh
+leis
+leis-san
+leoth'
+leotha
+leotha-san
+linn
+m'
+m'a
+ma
+mac
+man
+mar
+mas
+mathaid
+mi
+mis'
+mise
+mo
+mu
+mu 'n
+mun
+mur
+mura
+mus
+na
+na b'
+na bu
+na iad
+nach
+nad
+nam
+nan
+nar
+nas
+neo
+no
+nuair
+o
+o'n
+oir
+oirbh
+oirbh-se
+oirnn
+oirnne
+oirre
+on
+orm
+orm-sa
+ormsa
+orra
+orra-san
+orrasan
+ort
+os
+r'
+ri
+ribh
+rinn
+ris
+rithe
+rithe-se
+rium
+rium-sa
+riums'
+riumsa
+riut
+riuth'
+riutha
+riuthasan
+ro
+ro'n
+roimh
+roimhe
+romhainn
+romham
+romhpa
+ron
+ruibh
+ruinn
+ruinne
+sa
+san
+sann
+se
+seach
+seo
+seothach
+shin
+sibh
+sibh-se
+sibhse
+sin
+sineach
+sinn
+sinne
+siod
+siodach
+siud
+siudach
+sna # ann an
+sè
+t'
+tarsaing
+tarsainn
+tarsuinn
+thar
+thoigh
+thro
+thu
+thuc'
+thuca
+thugad
+thugaibh
+thugainn
+thugam
+thugamsa
+thuice
+thuige
+thus'
+thusa
+timcheall
+toigh
+toil
+tro
+tro' # troimh
+troimh
+troimhe
+tron
+tu
+tusa
+uair
+ud
+ugaibh
+ugam-s'
+ugam-sa
+uice
+uige
+uige-san
+umad
+unnta # ann an
+ur
+urrainn
+à
+às
+àsan
+á
+ás
+è
+ì
+ò
+ó
+""".split("\n")
+)

spacy/lang/grc/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "ἐρᾷ μὲν ἁγνὸς οὐρανὸς τρῶσαι χθόνα, ἔρως δὲ γαῖαν λαμβάνει γάμου τυχεῖν·",
     "εὐδαίμων Χαρίτων καὶ Μελάνιππος ἔφυ, θείας ἁγητῆρες ἐφαμερίοις φιλότατος.",

spacy/lang/gu/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "લોકશાહી એ સરકારનું એક એવું તંત્ર છે જ્યાં નાગરિકો મત દ્વારા સત્તાનો ઉપયોગ કરે છે.",
     "તે ગુજરાત રાજ્યના ધરમપુર શહેરમાં આવેલું હતું",

spacy/lang/he/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "סין מקימה קרן של 440 מיליון דולר להשקעה בהייטק בישראל",
     'רה"מ הודיע כי יחרים טקס בחסותו',

spacy/lang/hi/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "एप्पल 1 अरब डॉलर के लिए यू.के. स्टार्टअप खरीदने पर विचार कर रहा है।",
     "स्वायत्त कारें निर्माताओं की ओर बीमा दायित्व रखतीं हैं।",

spacy/lang/hr/lemma_lookup_license.txt

@@ -1,5 +1,5 @@
 The list of Croatian lemmas was extracted from the reldi-tagger repository (https://github.com/clarinsi/reldi-tagger).
-Reldi-tagger is licesned under the Apache 2.0 licence.
+Reldi-tagger is licensed under the Apache 2.0 licence.
 
 @InProceedings{ljubesic16-new,
   author = {Nikola Ljubešić and Filip Klubička and Željko Agić and Ivo-Pavao Jazbec},
@@ -12,4 +12,4 @@ Reldi-tagger is licesned under the Apache 2.0 licence.
   publisher = {European Language Resources Association (ELRA)},
   address = {Paris, France},
   isbn = {978-2-9517408-9-1}
- }
+ }

spacy/lang/hsb/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "To běšo wjelgin raźone a jo se wót luźi derje pśiwzeło. Tak som dožywiła wjelgin",
     "Jogo pśewóźowarce stej groniłej, až how w serbskich stronach njama Santa Claus nic pytaś.",

spacy/lang/ht/__init__.py

@@ -0,0 +1,55 @@
+from typing import Callable, Optional
+
+from thinc.api import Model
+
+from ...language import BaseDefaults, Language
+from .lemmatizer import HaitianCreoleLemmatizer
+from .lex_attrs import LEX_ATTRS
+from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES
+from .stop_words import STOP_WORDS
+from .syntax_iterators import SYNTAX_ITERATORS
+from .tag_map import TAG_MAP
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
+
+
+class HaitianCreoleDefaults(BaseDefaults):
+    tokenizer_exceptions = TOKENIZER_EXCEPTIONS
+    prefixes = TOKENIZER_PREFIXES
+    infixes = TOKENIZER_INFIXES
+    suffixes = TOKENIZER_SUFFIXES
+    lex_attr_getters = LEX_ATTRS
+    syntax_iterators = SYNTAX_ITERATORS
+    stop_words = STOP_WORDS
+    tag_map = TAG_MAP
+
+
+class HaitianCreole(Language):
+    lang = "ht"
+    Defaults = HaitianCreoleDefaults
+
+
+@HaitianCreole.factory(
+    "lemmatizer",
+    assigns=["token.lemma"],
+    default_config={
+        "model": None,
+        "mode": "rule",
+        "overwrite": False,
+        "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"},
+    },
+    default_score_weights={"lemma_acc": 1.0},
+)
+def make_lemmatizer(
+    nlp: Language,
+    model: Optional[Model],
+    name: str,
+    mode: str,
+    overwrite: bool,
+    scorer: Optional[Callable],
+):
+    return HaitianCreoleLemmatizer(
+        nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer
+    )
+
+
+__all__ = ["HaitianCreole"]

spacy/lang/ht/examples.py

@@ -0,0 +1,17 @@
+"""
+Example sentences to test spaCy and its language models.
+
+>>> from spacy.lang.ht.examples import sentences
+>>> docs = nlp.pipe(sentences)
+"""
+
+sentences = [
+    "Apple ap panse achte yon demaraj nan Wayòm Ini pou $1 milya dola",
+    "Machin otonòm fè responsablite asirans lan ale sou men fabrikan yo",
+    "San Francisco ap konsidere entèdi robo ki livre sou twotwa yo",
+    "Lond se yon gwo vil nan Wayòm Ini",
+    "Kote ou ye?",
+    "Kilès ki prezidan Lafrans?",
+    "Ki kapital Etazini?",
+    "Kile Barack Obama te fèt?",
+]

spacy/lang/ht/lemmatizer.py

@@ -0,0 +1,50 @@
+from typing import List, Tuple
+
+from ...pipeline import Lemmatizer
+from ...tokens import Token
+
+
+class HaitianCreoleLemmatizer(Lemmatizer):
+    """
+    Minimal Haitian Creole lemmatizer.
+    Returns a word's base form based on rules and lookup,
+    or defaults to the original form.
+    """
+
+    def is_base_form(self, token: Token) -> bool:
+        morph = token.morph.to_dict()
+        upos = token.pos_.lower()
+
+        # Consider unmarked forms to be base
+        if upos in {"noun", "verb", "adj", "adv"}:
+            if not morph:
+                return True
+            if upos == "noun" and morph.get("Number") == "Sing":
+                return True
+            if upos == "verb" and morph.get("VerbForm") == "Inf":
+                return True
+            if upos == "adj" and morph.get("Degree") == "Pos":
+                return True
+        return False
+
+    def rule_lemmatize(self, token: Token) -> List[str]:
+        string = token.text.lower()
+        pos = token.pos_.lower()
+        cache_key = (token.orth, token.pos)
+        if cache_key in self.cache:
+            return self.cache[cache_key]
+
+        forms = []
+
+        # fallback rule: just return lowercased form
+        forms.append(string)
+
+        self.cache[cache_key] = forms
+        return forms
+
+    @classmethod
+    def get_lookups_config(cls, mode: str) -> Tuple[List[str], List[str]]:
+        if mode == "rule":
+            required = ["lemma_lookup", "lemma_rules", "lemma_exc", "lemma_index"]
+            return (required, [])
+        return super().get_lookups_config(mode)

spacy/lang/ht/lex_attrs.py

@@ -0,0 +1,81 @@
+from ...attrs import LIKE_NUM, NORM
+
+# Cardinal numbers in Creole
+_num_words = set(
+    """
+zewo youn en de twa kat senk sis sèt uit nèf dis
+onz douz trèz katoz kenz sèz disèt dizwit diznèf
+vent trant karant sinkant swasant swasann-dis
+san mil milyon milya
+""".split()
+)
+
+# Ordinal numbers in Creole (some are French-influenced, some simplified)
+_ordinal_words = set(
+    """
+premye dezyèm twazyèm katryèm senkyèm sizyèm sètvyèm uitvyèm nèvyèm dizyèm
+onzèm douzyèm trèzyèm katozyèm kenzèm sèzyèm disetyèm dizwityèm diznèvyèm
+ventyèm trantyèm karantyèm sinkantyèm swasantyèm
+swasann-disyèm santyèm milyèm milyonnyèm milyadyèm
+""".split()
+)
+
+NORM_MAP = {
+    "'m": "mwen",
+    "'w": "ou",
+    "'l": "li",
+    "'n": "nou",
+    "'y": "yo",
+    "’m": "mwen",
+    "’w": "ou",
+    "’l": "li",
+    "’n": "nou",
+    "’y": "yo",
+    "m": "mwen",
+    "n": "nou",
+    "l": "li",
+    "y": "yo",
+    "w": "ou",
+    "t": "te",
+    "k": "ki",
+    "p": "pa",
+    "M": "Mwen",
+    "N": "Nou",
+    "L": "Li",
+    "Y": "Yo",
+    "W": "Ou",
+    "T": "Te",
+    "K": "Ki",
+    "P": "Pa",
+}
+
+
+def like_num(text):
+    text = text.strip().lower()
+    if text.startswith(("+", "-", "±", "~")):
+        text = text[1:]
+    text = text.replace(",", "").replace(".", "")
+    if text.isdigit():
+        return True
+    if text.count("/") == 1:
+        num, denom = text.split("/")
+        if num.isdigit() and denom.isdigit():
+            return True
+    if text in _num_words:
+        return True
+    if text in _ordinal_words:
+        return True
+    # Handle things like "3yèm", "10yèm", "25yèm", etc.
+    if text.endswith("yèm") and text[:-3].isdigit():
+        return True
+    return False
+
+
+def norm_custom(text):
+    return NORM_MAP.get(text, text.lower())
+
+
+LEX_ATTRS = {
+    LIKE_NUM: like_num,
+    NORM: norm_custom,
+}

spacy/lang/ht/punctuation.py

@@ -0,0 +1,58 @@
+from ..char_classes import (
+    ALPHA,
+    ALPHA_LOWER,
+    ALPHA_UPPER,
+    CONCAT_QUOTES,
+    HYPHENS,
+    LIST_ELLIPSES,
+    LIST_ICONS,
+    LIST_PUNCT,
+    LIST_QUOTES,
+    merge_chars,
+)
+
+ELISION = "'’".replace(" ", "")
+
+_prefixes_elision = "m n l y t k w"
+_prefixes_elision += " " + _prefixes_elision.upper()
+
+TOKENIZER_PREFIXES = (
+    LIST_PUNCT
+    + LIST_QUOTES
+    + [
+        r"(?:({pe})[{el}])(?=[{a}])".format(
+            a=ALPHA, el=ELISION, pe=merge_chars(_prefixes_elision)
+        )
+    ]
+)
+
+TOKENIZER_SUFFIXES = (
+    LIST_PUNCT
+    + LIST_QUOTES
+    + LIST_ELLIPSES
+    + [
+        r"(?<=[0-9])%",  # numbers like 10%
+        r"(?<=[0-9])(?:{h})".format(h=HYPHENS),  # hyphens after numbers
+        r"(?<=[{a}])['’]".format(a=ALPHA),  # apostrophes after letters
+        r"(?<=[{a}])['’][mwlnytk](?=\s|$)".format(a=ALPHA),  # contractions
+        r"(?<=[{a}0-9])\)",  # right parenthesis after letter/number
+        r"(?<=[{a}])\.(?=\s|$)".format(
+            a=ALPHA
+        ),  # period after letter if space or end of string
+        r"(?<=\))[\.\?!]",  # punctuation immediately after right parenthesis
+    ]
+)
+
+TOKENIZER_INFIXES = (
+    LIST_ELLIPSES
+    + LIST_ICONS
+    + [
+        r"(?<=[0-9])[+\-\*^](?=[0-9-])",
+        r"(?<=[{al}{q}])\.(?=[{au}{q}])".format(
+            al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES
+        ),
+        r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA),
+        r"(?<=[{a}0-9])(?:{h})(?=[{a}])".format(a=ALPHA, h=HYPHENS),
+        r"(?<=[{a}][{el}])(?=[{a}])".format(a=ALPHA, el=ELISION),
+    ]
+)

spacy/lang/ht/stop_words.py

@@ -0,0 +1,49 @@
+STOP_WORDS = set(
+    """
+a ak an ankò ant apre ap atò avan avanlè
+byen bò byenke
+
+chak
+
+de depi deja deja
+
+e en epi èske
+
+fò fòk
+
+gen genyen
+
+ki kisa kilès kote koukou konsa konbyen konn konnen kounye kouman
+
+la l laa le lè li lye lò
+
+m m' mwen
+
+nan nap nou n'
+
+ou oumenm
+
+pa paske pami pandan pito pou pral preske pwiske
+
+se selman si sou sòt
+
+ta tap tankou te toujou tou tan tout toutotan twòp tèl
+
+w w' wi wè
+
+y y' yo yon yonn
+
+non o oh eh
+
+sa san si swa si
+
+men mèsi oswa osinon
+
+""".split()
+)
+
+# Add common contractions, with and without apostrophe variants
+contractions = ["m'", "n'", "w'", "y'", "l'", "t'", "k'"]
+for apostrophe in ["'", "’", "‘"]:
+    for word in contractions:
+        STOP_WORDS.add(word.replace("'", apostrophe))

spacy/lang/ht/syntax_iterators.py

@@ -0,0 +1,74 @@
+from typing import Iterator, Tuple, Union
+
+from ...errors import Errors
+from ...symbols import NOUN, PRON, PROPN
+from ...tokens import Doc, Span
+
+
+def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]:
+    """
+    Detect base noun phrases from a dependency parse for Haitian Creole.
+    Works on both Doc and Span objects.
+    """
+
+    # Core nominal dependencies common in Haitian Creole
+    labels = [
+        "nsubj",
+        "obj",
+        "obl",
+        "nmod",
+        "appos",
+        "ROOT",
+    ]
+
+    # Modifiers to optionally include in chunk (to the right)
+    post_modifiers = ["compound", "flat", "flat:name", "fixed"]
+
+    doc = doclike.doc
+    if not doc.has_annotation("DEP"):
+        raise ValueError(Errors.E029)
+
+    np_deps = {doc.vocab.strings.add(label) for label in labels}
+    np_mods = {doc.vocab.strings.add(mod) for mod in post_modifiers}
+    conj_label = doc.vocab.strings.add("conj")
+    np_label = doc.vocab.strings.add("NP")
+    adp_pos = doc.vocab.strings.add("ADP")
+    cc_pos = doc.vocab.strings.add("CCONJ")
+
+    prev_end = -1
+    for i, word in enumerate(doclike):
+        if word.pos not in (NOUN, PROPN, PRON):
+            continue
+        if word.left_edge.i <= prev_end:
+            continue
+
+        if word.dep in np_deps:
+            right_end = word
+            # expand to include known modifiers to the right
+            for child in word.rights:
+                if child.dep in np_mods:
+                    right_end = child.right_edge
+                elif child.pos == NOUN:
+                    right_end = child.right_edge
+
+            left_index = word.left_edge.i
+            # Skip prepositions at the start
+            if word.left_edge.pos == adp_pos:
+                left_index += 1
+
+            prev_end = right_end.i
+            yield left_index, right_end.i + 1, np_label
+
+        elif word.dep == conj_label:
+            head = word.head
+            while head.dep == conj_label and head.head.i < head.i:
+                head = head.head
+            if head.dep in np_deps:
+                left_index = word.left_edge.i
+                if word.left_edge.pos == cc_pos:
+                    left_index += 1
+                prev_end = word.i
+                yield left_index, word.i + 1, np_label
+
+
+SYNTAX_ITERATORS = {"noun_chunks": noun_chunks}

spacy/lang/ht/tag_map.py

@@ -0,0 +1,39 @@
+from spacy.symbols import (
+    ADJ,
+    ADP,
+    ADV,
+    AUX,
+    CCONJ,
+    DET,
+    INTJ,
+    NOUN,
+    NUM,
+    PART,
+    PRON,
+    PROPN,
+    PUNCT,
+    SCONJ,
+    SYM,
+    VERB,
+    X,
+)
+
+TAG_MAP = {
+    "NOUN": {"pos": NOUN},
+    "VERB": {"pos": VERB},
+    "AUX": {"pos": AUX},
+    "ADJ": {"pos": ADJ},
+    "ADV": {"pos": ADV},
+    "PRON": {"pos": PRON},
+    "DET": {"pos": DET},
+    "ADP": {"pos": ADP},
+    "SCONJ": {"pos": SCONJ},
+    "CCONJ": {"pos": CCONJ},
+    "PART": {"pos": PART},
+    "INTJ": {"pos": INTJ},
+    "NUM": {"pos": NUM},
+    "PROPN": {"pos": PROPN},
+    "PUNCT": {"pos": PUNCT},
+    "SYM": {"pos": SYM},
+    "X": {"pos": X},
+}

spacy/lang/ht/tokenizer_exceptions.py

@@ -0,0 +1,126 @@
+from spacy.symbols import NORM, ORTH
+
+
+def make_variants(base, first_norm, second_orth, second_norm):
+    return {
+        base: [
+            {ORTH: base.split("'")[0] + "'", NORM: first_norm},
+            {ORTH: second_orth, NORM: second_norm},
+        ],
+        base.capitalize(): [
+            {
+                ORTH: base.split("'")[0].capitalize() + "'",
+                NORM: first_norm.capitalize(),
+            },
+            {ORTH: second_orth, NORM: second_norm},
+        ],
+    }
+
+
+TOKENIZER_EXCEPTIONS = {"Dr.": [{ORTH: "Dr."}]}
+
+# Apostrophe forms
+TOKENIZER_EXCEPTIONS.update(make_variants("m'ap", "mwen", "ap", "ap"))
+TOKENIZER_EXCEPTIONS.update(make_variants("n'ap", "nou", "ap", "ap"))
+TOKENIZER_EXCEPTIONS.update(make_variants("l'ap", "li", "ap", "ap"))
+TOKENIZER_EXCEPTIONS.update(make_variants("y'ap", "yo", "ap", "ap"))
+TOKENIZER_EXCEPTIONS.update(make_variants("m'te", "mwen", "te", "te"))
+TOKENIZER_EXCEPTIONS.update(make_variants("m'pral", "mwen", "pral", "pral"))
+TOKENIZER_EXCEPTIONS.update(make_variants("w'ap", "ou", "ap", "ap"))
+TOKENIZER_EXCEPTIONS.update(make_variants("k'ap", "ki", "ap", "ap"))
+TOKENIZER_EXCEPTIONS.update(make_variants("p'ap", "pa", "ap", "ap"))
+TOKENIZER_EXCEPTIONS.update(make_variants("t'ap", "te", "ap", "ap"))
+
+# Non-apostrophe contractions (with capitalized variants)
+TOKENIZER_EXCEPTIONS.update(
+    {
+        "map": [
+            {ORTH: "m", NORM: "mwen"},
+            {ORTH: "ap", NORM: "ap"},
+        ],
+        "Map": [
+            {ORTH: "M", NORM: "Mwen"},
+            {ORTH: "ap", NORM: "ap"},
+        ],
+        "lem": [
+            {ORTH: "le", NORM: "le"},
+            {ORTH: "m", NORM: "mwen"},
+        ],
+        "Lem": [
+            {ORTH: "Le", NORM: "Le"},
+            {ORTH: "m", NORM: "mwen"},
+        ],
+        "lew": [
+            {ORTH: "le", NORM: "le"},
+            {ORTH: "w", NORM: "ou"},
+        ],
+        "Lew": [
+            {ORTH: "Le", NORM: "Le"},
+            {ORTH: "w", NORM: "ou"},
+        ],
+        "nap": [
+            {ORTH: "n", NORM: "nou"},
+            {ORTH: "ap", NORM: "ap"},
+        ],
+        "Nap": [
+            {ORTH: "N", NORM: "Nou"},
+            {ORTH: "ap", NORM: "ap"},
+        ],
+        "lap": [
+            {ORTH: "l", NORM: "li"},
+            {ORTH: "ap", NORM: "ap"},
+        ],
+        "Lap": [
+            {ORTH: "L", NORM: "Li"},
+            {ORTH: "ap", NORM: "ap"},
+        ],
+        "yap": [
+            {ORTH: "y", NORM: "yo"},
+            {ORTH: "ap", NORM: "ap"},
+        ],
+        "Yap": [
+            {ORTH: "Y", NORM: "Yo"},
+            {ORTH: "ap", NORM: "ap"},
+        ],
+        "mte": [
+            {ORTH: "m", NORM: "mwen"},
+            {ORTH: "te", NORM: "te"},
+        ],
+        "Mte": [
+            {ORTH: "M", NORM: "Mwen"},
+            {ORTH: "te", NORM: "te"},
+        ],
+        "mpral": [
+            {ORTH: "m", NORM: "mwen"},
+            {ORTH: "pral", NORM: "pral"},
+        ],
+        "Mpral": [
+            {ORTH: "M", NORM: "Mwen"},
+            {ORTH: "pral", NORM: "pral"},
+        ],
+        "wap": [
+            {ORTH: "w", NORM: "ou"},
+            {ORTH: "ap", NORM: "ap"},
+        ],
+        "Wap": [
+            {ORTH: "W", NORM: "Ou"},
+            {ORTH: "ap", NORM: "ap"},
+        ],
+        "kap": [
+            {ORTH: "k", NORM: "ki"},
+            {ORTH: "ap", NORM: "ap"},
+        ],
+        "Kap": [
+            {ORTH: "K", NORM: "Ki"},
+            {ORTH: "ap", NORM: "ap"},
+        ],
+        "tap": [
+            {ORTH: "t", NORM: "te"},
+            {ORTH: "ap", NORM: "ap"},
+        ],
+        "Tap": [
+            {ORTH: "T", NORM: "Te"},
+            {ORTH: "ap", NORM: "ap"},
+        ],
+    }
+)

spacy/lang/hu/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Az Apple egy brit startup vásárlását tervezi 1 milliárd dollár értékben.",
     "San Francisco vezetése mérlegeli a járdát használó szállító robotok betiltását.",

spacy/lang/hu/punctuation.py

@@ -11,7 +11,7 @@ from ..char_classes import (
 )
 
 # removing ° from the special icons to keep e.g. 99° as one token
-_concat_icons = CONCAT_ICONS.replace("\u00B0", "")
+_concat_icons = CONCAT_ICONS.replace("\u00b0", "")
 
 _currency = r"\$¢£€¥฿"
 _quotes = CONCAT_QUOTES.replace("'", "")

spacy/lang/hy/examples.py

@@ -4,7 +4,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Լոնդոնը Միացյալ Թագավորության մեծ քաղաք է։",
     "Ո՞վ է Ֆրանսիայի նախագահը։",

spacy/lang/id/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Indonesia merupakan negara kepulauan yang kaya akan budaya.",
     "Berapa banyak warga yang dibutuhkan saat kerja bakti?",

spacy/lang/id/tokenizer_exceptions.py

@@ -156,7 +156,7 @@ for orth in [
     "S.T.",
     "S.T.Han",
     "S.Th.",
-    "S.Th.I" "S.TI.",
+    "S.Th.IS.TI.",
     "S.T.P.",
     "S.TrK",
     "S.Tekp.",
@@ -210,7 +210,7 @@ for orth in [
     "hlm.",
     "i/o",
     "n.b.",
-    "p.p." "pjs.",
+    "p.p.pjs.",
     "s.d.",
     "tel.",
     "u.p.",

spacy/lang/it/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "Apple vuole comprare una startup del Regno Unito per un miliardo di dollari",
     "Le automobili a guida autonoma spostano la responsabilità assicurativa verso i produttori",

spacy/lang/ja/__init__.py

@@ -32,7 +32,6 @@ split_mode = null
 """
 
 
-@registry.tokenizers("spacy.ja.JapaneseTokenizer")
 def create_tokenizer(split_mode: Optional[str] = None):
     def japanese_tokenizer_factory(nlp):
         return JapaneseTokenizer(nlp.vocab, split_mode=split_mode)
@@ -62,7 +61,7 @@ class JapaneseTokenizer(DummyTokenizer):
             zip(*dtokens) if dtokens else [[]] * 7
         )
         sub_tokens_list = list(sub_tokens_list)
-        doc = Doc(self.vocab, words=words, spaces=spaces)
+        doc = Doc(self.vocab, words=list(words), spaces=spaces)
         next_pos = None  # for bi-gram rules
         for idx, (token, dtoken) in enumerate(zip(doc, dtokens)):
             token.tag_ = dtoken.tag
@@ -103,9 +102,9 @@ class JapaneseTokenizer(DummyTokenizer):
                 token.dictionary_form(),  # lemma
                 token.normalized_form(),
                 token.reading_form(),
-                sub_tokens_list[idx]
-                if sub_tokens_list
-                else None,  # user_data['sub_tokens']
+                (
+                    sub_tokens_list[idx] if sub_tokens_list else None
+                ),  # user_data['sub_tokens']
             )
             for idx, token in enumerate(sudachipy_tokens)
             if len(token.surface()) > 0

spacy/lang/ja/examples.py

@@ -5,7 +5,6 @@ Example sentences to test spaCy and its language models.
 >>> docs = nlp.pipe(sentences)
 """
 
-
 sentences = [
     "アップルがイギリスの新興企業を１０億ドルで購入を検討",
     "自動運転車の損害賠償責任、自動車メーカーに一定の負担を求める",