Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Lise
|
dfe27516d9 | ||
|
Sofie Van Landeghem
|
c32c1289a9 | ||
|
Daniël de Kok
|
a36474600f | ||
|
Daniël de Kok
|
b25927251d | ||
|
Daniël de Kok
|
e1c1889acf | ||
|
Adriane Boyd
|
7174588155 |
@@ -10,7 +10,7 @@ SLACK_TOKEN = os.environ.get("SLACK_BOT_TOKEN", "ENV VAR not available!")
|
||||
DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
|
||||
|
||||
client = WebClient(SLACK_TOKEN)
|
||||
github_context = json.loads(sys.stdin.read())
|
||||
github_context = json.loads(sys.argv[1])
|
||||
|
||||
event = github_context['event']
|
||||
pr_title = event['pull_request']["title"]
|
||||
|
||||
@@ -1,114 +0,0 @@
|
||||
name: Build
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
# ytf did they invent their own syntax that's almost regex?
|
||||
# ** matches 'zero or more of any character'
|
||||
- 'release-v[0-9]+.[0-9]+.[0-9]+**'
|
||||
- 'prerelease-v[0-9]+.[0-9]+.[0-9]+**'
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
build_wheels:
|
||||
uses: explosion/gha-cibuildwheel/.github/workflows/cibuildwheel.yml@2c98f757f13d112cf73fcf4b627249f1fffb5aae # main
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
with:
|
||||
wheel-name-pattern: "spacy-*.whl"
|
||||
pure-python: false
|
||||
secrets:
|
||||
gh-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
smoke_test:
|
||||
name: Smoke test
|
||||
# No checkout here: the jobs below must import the installed wheel, and a
|
||||
# checked-out source tree in the working directory would shadow it.
|
||||
needs: build_wheels
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
# The release created by build_wheels is a draft, which can't be looked
|
||||
# up by tag, so pull the wheel from this run's artifacts instead
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
pattern: "cibw-wheels-ubuntu-latest-*"
|
||||
merge-multiple: true
|
||||
path: "dist"
|
||||
|
||||
- name: Install from wheel
|
||||
run: |
|
||||
WHEEL=$(ls dist/spacy-*cp312*manylinux*x86_64*.whl | head -1)
|
||||
pip install "$WHEEL"
|
||||
|
||||
- name: Test import
|
||||
run: python -c "import spacy; print('spacy==' + spacy.__version__)"
|
||||
|
||||
- name: Download and load model
|
||||
run: |
|
||||
python -m spacy download en_core_web_sm
|
||||
python -c "
|
||||
import spacy
|
||||
nlp = spacy.load('en_core_web_sm')
|
||||
doc = nlp('Apple is looking at buying U.K. startup for \$1 billion')
|
||||
assert len(doc.ents) > 0, 'No entities found'
|
||||
print('Model load OK:', nlp.meta['name'], '@', nlp.meta['version'])
|
||||
print('Entities:', [(ent.text, ent.label_) for ent in doc.ents])
|
||||
"
|
||||
|
||||
upgrade_test:
|
||||
name: Upgrade test
|
||||
# No checkout here: the jobs below must import the installed wheel, and a
|
||||
# checked-out source tree in the working directory would shadow it.
|
||||
needs: build_wheels
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install previous spaCy version
|
||||
run: |
|
||||
# strip the release-v/prerelease-v prefix to get a version specifier;
|
||||
# click is needed because spacy <=3.8.14 imports it without
|
||||
# declaring it (#13971) and modern typer no longer pulls it in
|
||||
pip install "spacy>=3.8.0,<${GITHUB_REF_NAME##*-v}" click || pip install "spacy<4" click
|
||||
python -m spacy download en_core_web_sm
|
||||
python -c "
|
||||
import spacy
|
||||
nlp = spacy.load('en_core_web_sm')
|
||||
print('Pre-upgrade:', spacy.__version__, nlp.meta['name'], '@', nlp.meta['version'])
|
||||
"
|
||||
|
||||
# The release created by build_wheels is a draft, which can't be looked
|
||||
# up by tag, so pull the wheel from this run's artifacts instead
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
pattern: "cibw-wheels-ubuntu-latest-*"
|
||||
merge-multiple: true
|
||||
path: "dist"
|
||||
|
||||
- name: Upgrade to new version
|
||||
run: |
|
||||
WHEEL=$(ls dist/spacy-*cp312*manylinux*x86_64*.whl | head -1)
|
||||
pip install "$WHEEL"
|
||||
|
||||
- name: Test model still loads after upgrade
|
||||
run: |
|
||||
python -c "
|
||||
import spacy
|
||||
nlp = spacy.load('en_core_web_sm')
|
||||
doc = nlp('Apple is looking at buying U.K. startup for \$1 billion')
|
||||
assert len(doc.ents) > 0, 'No entities found after upgrade'
|
||||
print('Post-upgrade:', spacy.__version__, nlp.meta['name'], '@', nlp.meta['version'])
|
||||
print('Entities:', [(ent.text, ent.label_) for ent in doc.ents])
|
||||
"
|
||||
@@ -6,8 +6,6 @@ on:
|
||||
- created
|
||||
- edited
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
explosion-bot:
|
||||
if: github.repository_owner == 'explosion'
|
||||
@@ -17,15 +15,13 @@ jobs:
|
||||
env:
|
||||
GITHUB_CONTEXT: ${{ toJson(github) }}
|
||||
run: echo "$GITHUB_CONTEXT"
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-python@v4
|
||||
- name: Install and run explosion-bot
|
||||
run: |
|
||||
git config --global url."https://x-access-token:${EXPLOSIONBOT_TOKEN}@github.com/".insteadOf "https://github.com/"
|
||||
pip install git+https://github.com/explosion/explosion-bot
|
||||
pip install git+https://${{ secrets.EXPLOSIONBOT_TOKEN }}@github.com/explosion/explosion-bot
|
||||
python -m explosionbot
|
||||
env:
|
||||
EXPLOSIONBOT_TOKEN: ${{ secrets.EXPLOSIONBOT_TOKEN }}
|
||||
INPUT_TOKEN: ${{ secrets.EXPLOSIONBOT_TOKEN }}
|
||||
INPUT_BK_TOKEN: ${{ secrets.BUILDKITE_SECRET }}
|
||||
ENABLED_COMMANDS: "test_gpu,test_slow,test_slow_gpu"
|
||||
|
||||
@@ -11,16 +11,12 @@ on:
|
||||
types:
|
||||
- labeled
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
issue-manager:
|
||||
permissions:
|
||||
issues: write
|
||||
if: github.repository_owner == 'explosion'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: tiangolo/issue-manager@4d1b7e05935a404dc8337d30bd23be46be8bb8e5 # 0.4.0
|
||||
- uses: tiangolo/issue-manager@0.4.0
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
config: >
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
if: github.repository_owner == 'explosion'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: dessant/lock-threads@1bf7ec25051fe7c00bdd17e6a7cf3d7bfb7dc771 # v5
|
||||
- uses: dessant/lock-threads@v4
|
||||
with:
|
||||
process-only: 'issues'
|
||||
issue-inactive-days: '30'
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
# The cibuildwheel action triggers on creation of a release, this
|
||||
# triggers on publication.
|
||||
# The expected workflow is to create a draft release and let the wheels
|
||||
# upload, and then hit 'publish', which uploads to PyPi.
|
||||
|
||||
on:
|
||||
release:
|
||||
types:
|
||||
- published
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
upload_pypi:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: pypi
|
||||
url: https://pypi.org/p/spacy
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
if: github.event_name == 'release' && github.event.action == 'published'
|
||||
# or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this)
|
||||
# if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
|
||||
steps:
|
||||
- uses: robinraju/release-downloader@daf26c55d821e836577a15f77d86ddc078948b05 # v1
|
||||
with:
|
||||
tag: ${{ github.event.release.tag_name }}
|
||||
fileName: '*'
|
||||
out-file-path: 'dist'
|
||||
- uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
ref: ${{ matrix.branch }}
|
||||
- name: Get commits from past 24 hours
|
||||
@@ -5,16 +5,21 @@ on:
|
||||
paths:
|
||||
- "website/meta/universe.json"
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: github.repository_owner == 'explosion'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
- name: Dump GitHub context
|
||||
env:
|
||||
GITHUB_CONTEXT: ${{ toJson(github) }}
|
||||
PR_NUMBER: ${{github.event.number}}
|
||||
run: |
|
||||
echo "$GITHUB_CONTEXT"
|
||||
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.10'
|
||||
- name: Install Bernadette app dependency and send an alert
|
||||
@@ -25,4 +30,4 @@ jobs:
|
||||
run: |
|
||||
pip install slack-sdk==3.17.2 aiohttp==3.8.1
|
||||
echo "$CHANNEL"
|
||||
echo "$GITHUB_CONTEXT" | python .github/spacy_universe_alert.py
|
||||
python .github/spacy_universe_alert.py "$GITHUB_CONTEXT"
|
||||
|
||||
+39
-22
@@ -2,8 +2,6 @@ name: tests
|
||||
|
||||
on:
|
||||
push:
|
||||
tags-ignore:
|
||||
- '**'
|
||||
branches-ignore:
|
||||
- "spacy.io"
|
||||
- "nightly.spacy.io"
|
||||
@@ -12,6 +10,7 @@ on:
|
||||
- "*.md"
|
||||
- "*.mdx"
|
||||
- "website/**"
|
||||
- ".github/workflows/**"
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened, edited]
|
||||
paths-ignore:
|
||||
@@ -19,9 +18,6 @@ on:
|
||||
- "*.mdx"
|
||||
- "website/**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
name: Validate
|
||||
@@ -29,40 +25,63 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out repo
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Configure Python version
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.10"
|
||||
python-version: "3.7"
|
||||
architecture: x64
|
||||
|
||||
- name: ruff format
|
||||
- name: black
|
||||
run: |
|
||||
python -m pip install ruff -c requirements.txt
|
||||
python -m ruff format spacy --check
|
||||
- name: ruff isort
|
||||
python -m pip install black -c requirements.txt
|
||||
python -m black spacy --check
|
||||
- name: isort
|
||||
run: |
|
||||
python -m ruff check spacy --select I
|
||||
python -m pip install isort -c requirements.txt
|
||||
python -m isort spacy --check
|
||||
- name: flake8
|
||||
run: |
|
||||
python -m pip install flake8==5.0.4
|
||||
python -m flake8 spacy --count --select=E901,E999,F821,F822,F823,W605 --show-source --statistics
|
||||
- name: cython-lint
|
||||
run: |
|
||||
python -m pip install cython-lint -c requirements.txt
|
||||
# E501: line too log, W291: trailing whitespace, E266: too many leading '#' for block comment
|
||||
cython-lint spacy --ignore E501,W291,E266
|
||||
|
||||
tests:
|
||||
name: Test
|
||||
needs: Validate
|
||||
strategy:
|
||||
fail-fast: false
|
||||
fail-fast: true
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-latest]
|
||||
python_version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||
python_version: ["3.12"]
|
||||
include:
|
||||
- os: windows-latest
|
||||
python_version: "3.7"
|
||||
- os: macos-latest
|
||||
python_version: "3.8"
|
||||
- os: ubuntu-latest
|
||||
python_version: "3.9"
|
||||
- os: windows-latest
|
||||
python_version: "3.10"
|
||||
- os: macos-latest
|
||||
python_version: "3.11"
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- name: Check out repo
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Configure Python version
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python_version }}
|
||||
architecture: x64
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
@@ -96,7 +115,7 @@ jobs:
|
||||
shell: bash
|
||||
|
||||
- name: Test import
|
||||
run: python -W error -W 'ignore:Core Pydantic V1:UserWarning:pydantic' -c "import spacy"
|
||||
run: python -W error -c "import spacy"
|
||||
|
||||
- name: "Test download CLI"
|
||||
run: |
|
||||
@@ -140,9 +159,7 @@ jobs:
|
||||
- name: "Test assemble CLI"
|
||||
run: |
|
||||
python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_sm'}; config.to_disk('ner_source_sm.cfg')"
|
||||
python -m spacy assemble ner_source_sm.cfg output_dir
|
||||
env:
|
||||
PYTHONWARNINGS: "error,ignore::DeprecationWarning"
|
||||
PYTHONWARNINGS="error,ignore::DeprecationWarning" python -m spacy assemble ner_source_sm.cfg output_dir
|
||||
if: matrix.python_version == '3.9'
|
||||
|
||||
- name: "Test assemble CLI vectors warning"
|
||||
@@ -157,7 +174,7 @@ jobs:
|
||||
|
||||
- name: "Run CPU tests"
|
||||
run: |
|
||||
python -m pytest --pyargs spacy -W error -W 'ignore:Core Pydantic V1:UserWarning:pydantic'
|
||||
python -m pytest --pyargs spacy -W error
|
||||
if: "!(startsWith(matrix.os, 'macos') && matrix.python_version == '3.11')"
|
||||
|
||||
- name: "Run CPU tests with thinc-apple-ops"
|
||||
|
||||
@@ -13,9 +13,6 @@ on:
|
||||
paths:
|
||||
- "website/meta/universe.json"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
name: Validate
|
||||
@@ -23,12 +20,13 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out repo
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Configure Python version
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.7"
|
||||
architecture: x64
|
||||
|
||||
- name: Validate website/meta/universe.json
|
||||
run: |
|
||||
|
||||
+11
-5
@@ -1,7 +1,13 @@
|
||||
repos:
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.9.0
|
||||
- repo: https://github.com/ambv/black
|
||||
rev: 22.3.0
|
||||
hooks:
|
||||
- id: ruff
|
||||
args: ['--fix']
|
||||
- id: ruff-format
|
||||
- id: black
|
||||
language_version: python3.7
|
||||
additional_dependencies: ['click==8.0.4']
|
||||
- repo: https://github.com/pycqa/flake8
|
||||
rev: 5.0.4
|
||||
hooks:
|
||||
- id: flake8
|
||||
args:
|
||||
- "--config=setup.cfg"
|
||||
|
||||
+7
-6
@@ -35,7 +35,7 @@ so that more people can benefit from it.
|
||||
|
||||
When opening an issue, use a **descriptive title** and include your
|
||||
**environment** (operating system, Python version, spaCy version). Our
|
||||
[issue templates](https://github.com/explosion/spaCy/issues/new/choose) help you
|
||||
[issue template](https://github.com/explosion/spaCy/issues/new) helps you
|
||||
remember the most important details to include. If you've discovered a bug, you
|
||||
can also submit a [regression test](#fixing-bugs) straight away. When you're
|
||||
opening an issue to report the bug, simply refer to your pull request in the
|
||||
@@ -449,12 +449,13 @@ and plugins in spaCy v3.0, and we can't wait to see what you build with it!
|
||||
[`spacy`](https://github.com/topics/spacy?o=desc&s=stars) and
|
||||
[`spacy-extensions`](https://github.com/topics/spacy-extension?o=desc&s=stars)
|
||||
to make it easier to find. Those are also the topics we're linking to from the
|
||||
spaCy website. If you're sharing your project on X, feel free to tag
|
||||
[@spacy_io](https://x.com/spacy_io) so we can check it out.
|
||||
spaCy website. If you're sharing your project on Twitter, feel free to tag
|
||||
[@spacy_io](https://twitter.com/spacy_io) so we can check it out.
|
||||
|
||||
- Once your extension is published, you can open a
|
||||
[PR](https://github.com/explosion/spaCy/pulls) to suggest it for the
|
||||
[Universe](https://spacy.io/universe) page.
|
||||
- Once your extension is published, you can open an issue on the
|
||||
[issue tracker](https://github.com/explosion/spacy/issues) to suggest it for the
|
||||
[resources directory](https://spacy.io/usage/resources#extensions) on the
|
||||
website.
|
||||
|
||||
📖 **For more tips and best practices, see the [checklist for developing spaCy extensions](https://spacy.io/usage/processing-pipelines#extensions).**
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (C) 2016-2024 ExplosionAI GmbH, 2016 spaCy GmbH, 2015 Matthew Honnibal
|
||||
Copyright (C) 2016-2023 ExplosionAI GmbH, 2016 spaCy GmbH, 2015 Matthew Honnibal
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@@ -4,6 +4,5 @@ include README.md
|
||||
include pyproject.toml
|
||||
include spacy/py.typed
|
||||
recursive-include spacy/cli *.yml
|
||||
recursive-include spacy/tests *.json
|
||||
recursive-include licenses *
|
||||
recursive-exclude spacy *.cpp
|
||||
|
||||
@@ -16,7 +16,7 @@ model packaging, deployment and workflow management. spaCy is commercial
|
||||
open-source software, released under the
|
||||
[MIT license](https://github.com/explosion/spaCy/blob/master/LICENSE).
|
||||
|
||||
💫 **Version 3.8 out now!**
|
||||
💫 **Version 3.7 out now!**
|
||||
[Check out the release notes here.](https://github.com/explosion/spaCy/releases)
|
||||
|
||||
[](https://github.com/explosion/spaCy/actions/workflows/tests.yml)
|
||||
@@ -28,6 +28,7 @@ open-source software, released under the
|
||||
<br />
|
||||
[](https://pypi.org/project/spacy/)
|
||||
[](https://anaconda.org/conda-forge/spacy)
|
||||
[](https://twitter.com/spacy_io)
|
||||
|
||||
## 📖 Documentation
|
||||
|
||||
@@ -46,7 +47,6 @@ open-source software, released under the
|
||||
| 👩🏫 **[Online Course]** | Learn spaCy in this free and interactive online course. |
|
||||
| 📰 **[Blog]** | Read about current spaCy and Prodigy development, releases, talks and more from Explosion. |
|
||||
| 📺 **[Videos]** | Our YouTube channel with video tutorials, talks and more. |
|
||||
| 🔴 **[Live Stream]** | Join Matt as he works on spaCy and chat about NLP. |
|
||||
| 🛠 **[Changelog]** | Changes and version history. |
|
||||
| 💝 **[Contribute]** | How to contribute to the spaCy project and code base. |
|
||||
| 👕 **[Swag]** | Support us and our work with unique, custom-designed swag! |
|
||||
@@ -62,7 +62,6 @@ open-source software, released under the
|
||||
[universe]: https://spacy.io/universe
|
||||
[spacy vs code extension]: https://github.com/explosion/spacy-vscode
|
||||
[videos]: https://www.youtube.com/c/ExplosionAI
|
||||
[live stream]: https://www.youtube.com/playlist?list=PLBmcuObd5An5_iAxNYLJa_xWmNzsYce8c
|
||||
[online course]: https://course.spacy.io
|
||||
[blog]: https://explosion.ai
|
||||
[project templates]: https://github.com/explosion/projects
|
||||
@@ -80,14 +79,13 @@ more people can benefit from it.
|
||||
| Type | Platforms |
|
||||
| ------------------------------- | --------------------------------------- |
|
||||
| 🚨 **Bug Reports** | [GitHub Issue Tracker] |
|
||||
| 🎁 **Feature Requests & Ideas** | [GitHub Discussions] · [Live Stream] |
|
||||
| 🎁 **Feature Requests & Ideas** | [GitHub Discussions] |
|
||||
| 👩💻 **Usage Questions** | [GitHub Discussions] · [Stack Overflow] |
|
||||
| 🗯 **General Discussion** | [GitHub Discussions] · [Live Stream] |
|
||||
| 🗯 **General Discussion** | [GitHub Discussions] |
|
||||
|
||||
[github issue tracker]: https://github.com/explosion/spaCy/issues
|
||||
[github discussions]: https://github.com/explosion/spaCy/discussions
|
||||
[stack overflow]: https://stackoverflow.com/questions/tagged/spacy
|
||||
[live stream]: https://www.youtube.com/playlist?list=PLBmcuObd5An5_iAxNYLJa_xWmNzsYce8c
|
||||
|
||||
## Features
|
||||
|
||||
@@ -117,7 +115,7 @@ For detailed installation instructions, see the
|
||||
|
||||
- **Operating system**: macOS / OS X · Linux · Windows (Cygwin, MinGW, Visual
|
||||
Studio)
|
||||
- **Python version**: Python >=3.7, <3.13 (only 64 bit)
|
||||
- **Python version**: Python 3.7+ (only 64 bit)
|
||||
- **Package managers**: [pip] · [conda] (via `conda-forge`)
|
||||
|
||||
[pip]: https://pypi.org/project/spacy/
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
Bug fix for model downloading in environments without pip on PATH
|
||||
|
||||
## Fixes
|
||||
|
||||
- Fix `spacy download` failing in environments where `pip` is not on PATH but is available as a Python module (e.g., some virtual environments and containers)
|
||||
@@ -1,20 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
# Insist repository is clean
|
||||
git diff-index --quiet HEAD
|
||||
|
||||
version=$(grep "__version__ = " spacy/about.py)
|
||||
version=${version/__version__ = }
|
||||
version=${version/\'/}
|
||||
version=${version/\'/}
|
||||
version=${version/\"/}
|
||||
version=${version/\"/}
|
||||
|
||||
echo "Pushing release-v"$version
|
||||
|
||||
git tag -d release-v$version || true
|
||||
git push origin :release-v$version || true
|
||||
git tag release-v$version
|
||||
git push origin release-v$version
|
||||
@@ -1,2 +1,6 @@
|
||||
# build version constraints for use with wheelwright
|
||||
numpy>=2.0.0,<3.0.0
|
||||
numpy==1.15.0; python_version=='3.7' and platform_machine!='aarch64'
|
||||
numpy==1.19.2; python_version=='3.7' and platform_machine=='aarch64'
|
||||
numpy==1.17.3; python_version=='3.8' and platform_machine!='aarch64'
|
||||
numpy==1.19.2; python_version=='3.8' and platform_machine=='aarch64'
|
||||
numpy>=1.25.0; python_version>='3.9'
|
||||
|
||||
+6
-67
@@ -1,76 +1,15 @@
|
||||
[build-system]
|
||||
requires = [
|
||||
"setuptools",
|
||||
"cython>=3.0,<4.0",
|
||||
"cython>=0.25,<3.0",
|
||||
"cymem>=2.0.2,<2.1.0",
|
||||
"preshed>=3.0.2,<3.1.0",
|
||||
"murmurhash>=0.28.0,<1.1.0",
|
||||
"thinc>=8.3.12,<8.4.0",
|
||||
"numpy>=2.0.0,<3.0.0"
|
||||
"thinc>=8.2.2,<8.3.0",
|
||||
"numpy>=1.15.0; python_version < '3.9'",
|
||||
"numpy>=1.25.0; python_version >= '3.9'",
|
||||
]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[tool.cibuildwheel]
|
||||
build = "*"
|
||||
skip = "cp39* *-win32 *i686* cp3??t-* *cp310-win_arm64"
|
||||
test-skip = ""
|
||||
|
||||
archs = ["native"]
|
||||
|
||||
build-frontend = "default"
|
||||
config-settings = {}
|
||||
dependency-versions = "pinned"
|
||||
environment = { PIP_CONSTRAINT = "build-constraints.txt" }
|
||||
|
||||
environment-pass = []
|
||||
build-verbosity = 0
|
||||
|
||||
before-all = "curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain stable"
|
||||
before-build = "pip install -r requirements.txt && python setup.py clean"
|
||||
repair-wheel-command = ""
|
||||
|
||||
test-command = ""
|
||||
before-test = ""
|
||||
test-requires = []
|
||||
test-extras = []
|
||||
|
||||
container-engine = "docker"
|
||||
|
||||
# numpy >=2.3 only ships manylinux_2_28 wheels, so the build container must
|
||||
# be at least that; i686 keeps manylinux2014 (no 2_28 image) but isn't built
|
||||
manylinux-x86_64-image = "manylinux_2_28"
|
||||
manylinux-i686-image = "manylinux2014"
|
||||
manylinux-aarch64-image = "manylinux_2_28"
|
||||
manylinux-ppc64le-image = "manylinux2014"
|
||||
manylinux-s390x-image = "manylinux2014"
|
||||
manylinux-pypy_x86_64-image = "manylinux2014"
|
||||
manylinux-pypy_i686-image = "manylinux2014"
|
||||
manylinux-pypy_aarch64-image = "manylinux2014"
|
||||
|
||||
musllinux-x86_64-image = "musllinux_1_2"
|
||||
musllinux-i686-image = "musllinux_1_2"
|
||||
musllinux-aarch64-image = "musllinux_1_2"
|
||||
musllinux-ppc64le-image = "musllinux_1_2"
|
||||
musllinux-s390x-image = "musllinux_1_2"
|
||||
|
||||
[tool.cibuildwheel.linux]
|
||||
repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}"
|
||||
|
||||
[tool.cibuildwheel.macos]
|
||||
repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}"
|
||||
|
||||
[tool.cibuildwheel.windows]
|
||||
|
||||
[tool.cibuildwheel.pyodide]
|
||||
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 88
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "W", "C", "B", "B9"]
|
||||
ignore = ["E203", "E266", "E501", "E731", "E741", "F541"]
|
||||
|
||||
[tool.ruff.lint.isort]
|
||||
combine-as-imports = true
|
||||
split-on-trailing-comma = true
|
||||
[tool.isort]
|
||||
profile = "black"
|
||||
|
||||
+17
-15
@@ -3,38 +3,40 @@ spacy-legacy>=3.0.11,<3.1.0
|
||||
spacy-loggers>=1.0.0,<2.0.0
|
||||
cymem>=2.0.2,<2.1.0
|
||||
preshed>=3.0.2,<3.1.0
|
||||
thinc>=8.3.12,<8.4.0
|
||||
ml_datasets>=0.2.1,<0.3.0
|
||||
thinc>=8.2.2,<8.3.0
|
||||
ml_datasets>=0.2.0,<0.3.0
|
||||
murmurhash>=0.28.0,<1.1.0
|
||||
wasabi>=0.9.1,<1.2.0
|
||||
srsly>=2.5.3,<3.0.0
|
||||
srsly>=2.4.3,<3.0.0
|
||||
catalogue>=2.0.6,<2.1.0
|
||||
typer>=0.3.0,<1.0.0
|
||||
click>=8.2.1,<9.0.0
|
||||
weasel>=1.0.0,<2.0.0
|
||||
typer>=0.3.0,<0.10.0
|
||||
smart-open>=5.2.1,<7.0.0
|
||||
weasel>=0.1.0,<0.4.0
|
||||
# Third party dependencies
|
||||
numpy>=2.0.0,<3.0.0
|
||||
numpy>=1.15.0; python_version < "3.9"
|
||||
numpy>=1.19.0; python_version >= "3.9"
|
||||
requests>=2.13.0,<3.0.0
|
||||
tqdm>=4.38.0,<5.0.0
|
||||
pydantic>=2.0.0,<3.0.0
|
||||
pydantic>=1.7.4,!=1.8,!=1.8.1,<3.0.0
|
||||
jinja2
|
||||
langcodes>=3.2.0,<4.0.0
|
||||
# Official Python utilities
|
||||
setuptools
|
||||
packaging>=20.0
|
||||
typing_extensions>=3.7.4.1,<4.5.0; python_version < "3.8"
|
||||
# Development dependencies
|
||||
pre-commit>=2.13.0
|
||||
cython>=3.0,<4.0
|
||||
cython>=0.25,<3.0
|
||||
pytest>=5.2.0,!=7.1.0
|
||||
pytest-timeout>=1.3.0,<2.0.0
|
||||
mock>=2.0.0,<3.0.0
|
||||
# hypothesis >=6.156 ships a Rust extension with no win_arm64 wheels, which
|
||||
# breaks wheel builds on windows-11-arm (sdist needs maturin)
|
||||
hypothesis>=3.27.0,<6.156.0
|
||||
mypy>=1.20.2,<1.21.0; platform_machine != "aarch64" and python_version >= "3.8"
|
||||
flake8>=3.8.0,<6.0.0
|
||||
hypothesis>=3.27.0,<7.0.0
|
||||
mypy>=1.5.0,<1.6.0; platform_machine != "aarch64" and python_version >= "3.8"
|
||||
types-mock>=0.1.1
|
||||
types-setuptools>=57.0.0
|
||||
types-requests
|
||||
types-setuptools>=57.0.0
|
||||
ruff>=0.9.0
|
||||
black==22.3.0
|
||||
cython-lint>=0.15.0
|
||||
confection>=1.3.2,<2.0.0
|
||||
isort>=5.0,<6.0
|
||||
|
||||
@@ -17,12 +17,11 @@ classifiers =
|
||||
Operating System :: Microsoft :: Windows
|
||||
Programming Language :: Cython
|
||||
Programming Language :: Python :: 3
|
||||
Programming Language :: Python :: 3.7
|
||||
Programming Language :: Python :: 3.8
|
||||
Programming Language :: Python :: 3.9
|
||||
Programming Language :: Python :: 3.10
|
||||
Programming Language :: Python :: 3.11
|
||||
Programming Language :: Python :: 3.12
|
||||
Programming Language :: Python :: 3.13
|
||||
Programming Language :: Python :: 3.14
|
||||
Topic :: Scientific/Engineering
|
||||
project_urls =
|
||||
Release notes = https://github.com/explosion/spaCy/releases
|
||||
@@ -31,18 +30,18 @@ project_urls =
|
||||
[options]
|
||||
zip_safe = false
|
||||
include_package_data = true
|
||||
python_requires = >=3.9,<3.15
|
||||
python_requires = >=3.7
|
||||
# NOTE: This section is superseded by pyproject.toml and will be removed in
|
||||
# spaCy v4
|
||||
setup_requires =
|
||||
cython>=3.0,<4.0
|
||||
numpy>=2.0.0,<3.0.0; python_version < "3.9"
|
||||
numpy>=2.0.0,<3.0.0; python_version >= "3.9"
|
||||
cython>=0.25,<3.0
|
||||
numpy>=1.15.0; python_version < "3.9"
|
||||
numpy>=1.19.0; python_version >= "3.9"
|
||||
# We also need our Cython packages here to compile against
|
||||
cymem>=2.0.2,<2.1.0
|
||||
preshed>=3.0.2,<3.1.0
|
||||
murmurhash>=0.28.0,<1.1.0
|
||||
thinc>=8.3.12,<8.4.0
|
||||
thinc>=8.2.2,<8.3.0
|
||||
install_requires =
|
||||
# Our libraries
|
||||
spacy-legacy>=3.0.11,<3.1.0
|
||||
@@ -50,24 +49,25 @@ install_requires =
|
||||
murmurhash>=0.28.0,<1.1.0
|
||||
cymem>=2.0.2,<2.1.0
|
||||
preshed>=3.0.2,<3.1.0
|
||||
thinc>=8.3.12,<8.4.0
|
||||
thinc>=8.2.2,<8.3.0
|
||||
wasabi>=0.9.1,<1.2.0
|
||||
srsly>=2.5.3,<3.0.0
|
||||
srsly>=2.4.3,<3.0.0
|
||||
catalogue>=2.0.6,<2.1.0
|
||||
weasel>=1.0.0,<2.0.0
|
||||
confection>=1.3.2,<2.0.0
|
||||
weasel>=0.1.0,<0.4.0
|
||||
# Third-party dependencies
|
||||
typer>=0.3.0,<1.0.0
|
||||
click>=8.2.1,<9.0.0
|
||||
typer>=0.3.0,<0.10.0
|
||||
smart-open>=5.2.1,<7.0.0
|
||||
tqdm>=4.38.0,<5.0.0
|
||||
numpy>=1.15.0; python_version < "3.9"
|
||||
numpy>=1.19.0; python_version >= "3.9"
|
||||
requests>=2.13.0,<3.0.0
|
||||
pydantic>=2.0.0,<3.0.0
|
||||
pydantic>=1.7.4,!=1.8,!=1.8.1,<3.0.0
|
||||
jinja2
|
||||
# Official Python utilities
|
||||
setuptools
|
||||
packaging>=20.0
|
||||
typing_extensions>=3.7.4.1,<4.5.0; python_version < "3.8"
|
||||
langcodes>=3.2.0,<4.0.0
|
||||
|
||||
[options.entry_points]
|
||||
console_scripts =
|
||||
@@ -117,7 +117,7 @@ cuda12x =
|
||||
cuda-autodetect =
|
||||
cupy-wheel>=11.0.0,<13.0.0
|
||||
apple =
|
||||
thinc-apple-ops>=1.0.0,<2.0.0
|
||||
thinc-apple-ops>=0.1.0.dev0,<1.0.0
|
||||
# Language tokenizers with external dependencies
|
||||
ja =
|
||||
sudachipy>=0.5.2,!=0.6.1
|
||||
@@ -133,13 +133,20 @@ universal = false
|
||||
[sdist]
|
||||
formats = gztar
|
||||
|
||||
[flake8]
|
||||
ignore = E203, E266, E501, E731, W503, E741, F541
|
||||
max-line-length = 80
|
||||
select = B,C,E,F,W,T4,B9
|
||||
exclude =
|
||||
.env,
|
||||
.git,
|
||||
__pycache__,
|
||||
_tokenizer_exceptions_list.py,
|
||||
|
||||
[tool:pytest]
|
||||
markers =
|
||||
slow: mark a test as slow
|
||||
issue: reference specific issue
|
||||
filterwarnings =
|
||||
error
|
||||
ignore:Core Pydantic V1:UserWarning:pydantic
|
||||
|
||||
[mypy]
|
||||
ignore_missing_imports = True
|
||||
|
||||
@@ -82,9 +82,9 @@ COMPILER_DIRECTIVES = {
|
||||
}
|
||||
# Files to copy into the package that are otherwise not included
|
||||
COPY_FILES = {
|
||||
ROOT / "setup.cfg": PACKAGE_ROOT / "tests" / "package" / "test.cfg",
|
||||
ROOT / "pyproject.toml": PACKAGE_ROOT / "tests" / "package" / "test.toml",
|
||||
ROOT / "requirements.txt": PACKAGE_ROOT / "tests" / "package" / "test.txt",
|
||||
ROOT / "setup.cfg": PACKAGE_ROOT / "tests" / "package",
|
||||
ROOT / "pyproject.toml": PACKAGE_ROOT / "tests" / "package",
|
||||
ROOT / "requirements.txt": PACKAGE_ROOT / "tests" / "package",
|
||||
}
|
||||
|
||||
|
||||
@@ -173,10 +173,10 @@ def setup_package():
|
||||
about = {}
|
||||
exec(f.read(), about)
|
||||
|
||||
for copy_file, target_file in COPY_FILES.items():
|
||||
for copy_file, target_dir in COPY_FILES.items():
|
||||
if copy_file.exists():
|
||||
shutil.copyfile(str(copy_file), str(target_file))
|
||||
print(f"Copied {copy_file} -> {target_file}")
|
||||
shutil.copy(str(copy_file), str(target_dir))
|
||||
print(f"Copied {copy_file} -> {target_dir}")
|
||||
|
||||
include_dirs = [
|
||||
numpy.get_include(),
|
||||
|
||||
+2
-25
@@ -10,39 +10,16 @@ setup_default_warnings() # noqa: E402
|
||||
# These are imported as part of the API
|
||||
from thinc.api import Config, prefer_gpu, require_cpu, require_gpu # noqa: F401
|
||||
|
||||
from . import (
|
||||
pipeline, # noqa: F401
|
||||
util,
|
||||
)
|
||||
from . import pipeline # noqa: F401
|
||||
from . import util
|
||||
from .about import __version__ # noqa: F401
|
||||
from .cli.info import info # noqa: F401
|
||||
from .errors import Errors
|
||||
from .glossary import explain # noqa: F401
|
||||
from .language import Language
|
||||
from .registrations import REGISTRY_POPULATED, populate_registry
|
||||
|
||||
# Rebuild pydantic v2 schemas that use forward references to Language/Vocab
|
||||
from .schemas import ( # noqa: F401
|
||||
ConfigSchema,
|
||||
ConfigSchemaInit,
|
||||
ConfigSchemaNlp,
|
||||
ConfigSchemaPretrain,
|
||||
ConfigSchemaTraining,
|
||||
)
|
||||
from .training import Example # noqa: F401
|
||||
from .util import logger, registry # noqa: F401
|
||||
from .vocab import Vocab
|
||||
|
||||
_rebuild_ns = {"Language": Language, "Vocab": Vocab, "Example": Example}
|
||||
for _schema in (
|
||||
ConfigSchemaTraining,
|
||||
ConfigSchemaNlp,
|
||||
ConfigSchemaPretrain,
|
||||
ConfigSchemaInit,
|
||||
ConfigSchema,
|
||||
):
|
||||
_schema.model_rebuild(_types_namespace=_rebuild_ns) # type: ignore[attr-defined]
|
||||
|
||||
if sys.maxunicode == 65535:
|
||||
raise SystemError(Errors.E130)
|
||||
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
# fmt: off
|
||||
__title__ = "spacy"
|
||||
__version__ = "3.8.15"
|
||||
__version__ = "3.7.2"
|
||||
__download_url__ = "https://github.com/explosion/spacy-models/releases/download"
|
||||
__compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
from wasabi import msg
|
||||
|
||||
# Needed for testing
|
||||
from . import download as download_module # noqa: F401
|
||||
from ._util import app, setup_cli # noqa: F401
|
||||
from .apply import apply # noqa: F401
|
||||
from .assemble import assemble_cli # noqa: F401
|
||||
|
||||
+16
-5
@@ -1,11 +1,15 @@
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from configparser import InterpolationError
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Dict,
|
||||
Iterable,
|
||||
List,
|
||||
Optional,
|
||||
Tuple,
|
||||
@@ -16,18 +20,23 @@ from typing import (
|
||||
import srsly
|
||||
import typer
|
||||
from click import NoSuchOption
|
||||
from click.shell_completion import split_arg_string
|
||||
from thinc.api import ConfigValidationError, require_gpu
|
||||
from click.parser import split_arg_string
|
||||
from thinc.api import Config, ConfigValidationError, require_gpu
|
||||
from thinc.util import gpu_is_available
|
||||
from typer.main import get_command
|
||||
from wasabi import Printer, msg
|
||||
from weasel import app as project_cli
|
||||
|
||||
from .. import about
|
||||
from ..compat import Literal
|
||||
from ..schemas import validate
|
||||
from ..util import (
|
||||
ENV_VARS,
|
||||
SimpleFrozenDict,
|
||||
import_file,
|
||||
is_compatible_version,
|
||||
logger,
|
||||
make_tempdir,
|
||||
registry,
|
||||
run_command,
|
||||
)
|
||||
@@ -59,7 +68,7 @@ INIT_HELP = """Commands for initializing configs and pipeline packages."""
|
||||
Arg = typer.Argument
|
||||
Opt = typer.Option
|
||||
|
||||
app = typer.Typer(name=NAME, help=HELP, rich_markup_mode=None)
|
||||
app = typer.Typer(name=NAME, help=HELP)
|
||||
benchmark_cli = typer.Typer(name="benchmark", help=BENCHMARK_HELP, no_args_is_help=True)
|
||||
debug_cli = typer.Typer(name="debug", help=DEBUG_HELP, no_args_is_help=True)
|
||||
init_cli = typer.Typer(name="init", help=INIT_HELP, no_args_is_help=True)
|
||||
@@ -216,11 +225,13 @@ def get_git_version(
|
||||
|
||||
|
||||
@overload
|
||||
def string_to_list(value: str, intify: Literal[False] = ...) -> List[str]: ...
|
||||
def string_to_list(value: str, intify: Literal[False] = ...) -> List[str]:
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def string_to_list(value: str, intify: Literal[True]) -> List[int]: ...
|
||||
def string_to_list(value: str, intify: Literal[True]) -> List[int]:
|
||||
...
|
||||
|
||||
|
||||
def string_to_list(value: str, intify: bool = False) -> Union[List[str], List[int]]:
|
||||
|
||||
+6
-9
@@ -22,7 +22,7 @@ to be grabbed ("text" by default)."""
|
||||
|
||||
out_help = "Path to save the resulting .spacy file"
|
||||
code_help = (
|
||||
"Path to Python file with additional code (registered functions) to be imported"
|
||||
"Path to Python file with additional " "code (registered functions) to be imported"
|
||||
)
|
||||
gold_help = "Use gold preprocessing provided in the .spacy files"
|
||||
force_msg = (
|
||||
@@ -72,15 +72,11 @@ def apply_cli(
|
||||
data_path: Path = Arg(..., help=path_help, exists=True),
|
||||
output_file: Path = Arg(..., help=out_help, dir_okay=False),
|
||||
code_path: Optional[Path] = Opt(None, "--code", "-c", help=code_help),
|
||||
text_key: str = Opt(
|
||||
"text", "--text-key", "-tk", help="Key containing text string for JSONL"
|
||||
),
|
||||
force_overwrite: bool = Opt(
|
||||
False, "--force", "-F", help="Force overwriting the output file"
|
||||
),
|
||||
text_key: str = Opt("text", "--text-key", "-tk", help="Key containing text string for JSONL"),
|
||||
force_overwrite: bool = Opt(False, "--force", "-F", help="Force overwriting the output file"),
|
||||
use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU."),
|
||||
batch_size: int = Opt(1, "--batch-size", "-b", help="Batch size."),
|
||||
n_process: int = Opt(1, "--n-process", "-n", help="number of processors to use."),
|
||||
n_process: int = Opt(1, "--n-process", "-n", help="number of processors to use.")
|
||||
):
|
||||
"""
|
||||
Apply a trained pipeline to documents to get predictions.
|
||||
@@ -118,7 +114,8 @@ def apply(
|
||||
if len(paths) == 0:
|
||||
docbin.to_disk(output_file)
|
||||
msg.warn(
|
||||
f"Did not find data to process, {data_path} seems to be an empty directory."
|
||||
"Did not find data to process,"
|
||||
f" {data_path} seems to be an empty directory."
|
||||
)
|
||||
return
|
||||
nlp = load_model(model)
|
||||
|
||||
+4
-19
@@ -24,25 +24,10 @@ from ._util import (
|
||||
def assemble_cli(
|
||||
# fmt: off
|
||||
ctx: typer.Context, # This is only used to read additional arguments
|
||||
config_path: Path = Arg(
|
||||
..., help="Path to config file", exists=True, allow_dash=True
|
||||
),
|
||||
output_path: Path = Arg(
|
||||
..., help="Output directory to store assembled pipeline in"
|
||||
),
|
||||
code_path: Optional[Path] = Opt(
|
||||
None,
|
||||
"--code",
|
||||
"-c",
|
||||
help="Path to Python file with additional code (registered functions) to be imported",
|
||||
),
|
||||
verbose: bool = Opt(
|
||||
False,
|
||||
"--verbose",
|
||||
"-V",
|
||||
"-VV",
|
||||
help="Display more information for debugging purposes",
|
||||
),
|
||||
config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
|
||||
output_path: Path = Arg(..., help="Output directory to store assembled pipeline in"),
|
||||
code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
|
||||
verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
|
||||
# fmt: on
|
||||
):
|
||||
"""
|
||||
|
||||
@@ -24,29 +24,13 @@ def benchmark_speed_cli(
|
||||
# fmt: off
|
||||
ctx: typer.Context,
|
||||
model: str = Arg(..., help="Model name or path"),
|
||||
data_path: Path = Arg(
|
||||
..., help="Location of binary evaluation data in .spacy format", exists=True
|
||||
),
|
||||
batch_size: Optional[int] = Opt(
|
||||
None, "--batch-size", "-b", min=1, help="Override the pipeline batch size"
|
||||
),
|
||||
data_path: Path = Arg(..., help="Location of binary evaluation data in .spacy format", exists=True),
|
||||
batch_size: Optional[int] = Opt(None, "--batch-size", "-b", min=1, help="Override the pipeline batch size"),
|
||||
no_shuffle: bool = Opt(False, "--no-shuffle", help="Do not shuffle benchmark data"),
|
||||
use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
|
||||
n_batches: int = Opt(
|
||||
50,
|
||||
"--batches",
|
||||
help="Minimum number of batches to benchmark",
|
||||
min=30,
|
||||
),
|
||||
warmup_epochs: int = Opt(
|
||||
3, "--warmup", "-w", min=0, help="Number of iterations over the data for warmup"
|
||||
),
|
||||
code_path: Optional[Path] = Opt(
|
||||
None,
|
||||
"--code",
|
||||
"-c",
|
||||
help="Path to Python file with additional code (registered functions) to be imported",
|
||||
),
|
||||
n_batches: int = Opt(50, "--batches", help="Minimum number of batches to benchmark", min=30,),
|
||||
warmup_epochs: int = Opt(3, "--warmup", "-w", min=0, help="Number of iterations over the data for warmup"),
|
||||
code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
|
||||
# fmt: on
|
||||
):
|
||||
"""
|
||||
@@ -167,7 +151,7 @@ def print_mean_with_ci(sample: numpy.ndarray):
|
||||
low = bootstrap_means[int(len(bootstrap_means) * 0.025)]
|
||||
high = bootstrap_means[int(len(bootstrap_means) * 0.975)]
|
||||
|
||||
print(f"Mean: {mean:.1f} words/s (95% CI: {low - mean:.1f} +{high - mean:.1f})")
|
||||
print(f"Mean: {mean:.1f} words/s (95% CI: {low-mean:.1f} +{high-mean:.1f})")
|
||||
|
||||
|
||||
def print_outliers(sample: numpy.ndarray):
|
||||
|
||||
+11
-41
@@ -48,47 +48,17 @@ class FileTypes(str, Enum):
|
||||
def convert_cli(
|
||||
# fmt: off
|
||||
input_path: str = Arg(..., help="Input file or directory", exists=True),
|
||||
output_dir: Path = Arg(
|
||||
"-", help="Output directory. '-' for stdout.", allow_dash=True, exists=True
|
||||
),
|
||||
file_type: FileTypes = Opt(
|
||||
"spacy", "--file-type", "-t", help="Type of data to produce"
|
||||
),
|
||||
n_sents: int = Opt(
|
||||
1, "--n-sents", "-n", help="Number of sentences per doc (0 to disable)"
|
||||
),
|
||||
seg_sents: bool = Opt(
|
||||
False, "--seg-sents", "-s", help="Segment sentences (for -c ner)"
|
||||
),
|
||||
model: Optional[str] = Opt(
|
||||
None,
|
||||
"--model",
|
||||
"--base",
|
||||
"-b",
|
||||
help="Trained spaCy pipeline for sentence segmentation to use as base (for --seg-sents)",
|
||||
),
|
||||
morphology: bool = Opt(
|
||||
False, "--morphology", "-m", help="Enable appending morphology to tags"
|
||||
),
|
||||
merge_subtokens: bool = Opt(
|
||||
False, "--merge-subtokens", "-T", help="Merge CoNLL-U subtokens"
|
||||
),
|
||||
converter: str = Opt(
|
||||
AUTO, "--converter", "-c", help=f"Converter: {tuple(CONVERTERS.keys())}"
|
||||
),
|
||||
ner_map: Optional[Path] = Opt(
|
||||
None,
|
||||
"--ner-map",
|
||||
"-nm",
|
||||
help="NER tag mapping (as JSON-encoded dict of entity types)",
|
||||
exists=True,
|
||||
),
|
||||
lang: Optional[str] = Opt(
|
||||
None, "--lang", "-l", help="Language (if tokenizer required)"
|
||||
),
|
||||
concatenate: bool = Opt(
|
||||
None, "--concatenate", "-C", help="Concatenate output to a single file"
|
||||
),
|
||||
output_dir: Path = Arg("-", help="Output directory. '-' for stdout.", allow_dash=True, exists=True),
|
||||
file_type: FileTypes = Opt("spacy", "--file-type", "-t", help="Type of data to produce"),
|
||||
n_sents: int = Opt(1, "--n-sents", "-n", help="Number of sentences per doc (0 to disable)"),
|
||||
seg_sents: bool = Opt(False, "--seg-sents", "-s", help="Segment sentences (for -c ner)"),
|
||||
model: Optional[str] = Opt(None, "--model", "--base", "-b", help="Trained spaCy pipeline for sentence segmentation to use as base (for --seg-sents)"),
|
||||
morphology: bool = Opt(False, "--morphology", "-m", help="Enable appending morphology to tags"),
|
||||
merge_subtokens: bool = Opt(False, "--merge-subtokens", "-T", help="Merge CoNLL-U subtokens"),
|
||||
converter: str = Opt(AUTO, "--converter", "-c", help=f"Converter: {tuple(CONVERTERS.keys())}"),
|
||||
ner_map: Optional[Path] = Opt(None, "--ner-map", "-nm", help="NER tag mapping (as JSON-encoded dict of entity types)", exists=True),
|
||||
lang: Optional[str] = Opt(None, "--lang", "-l", help="Language (if tokenizer required)"),
|
||||
concatenate: bool = Opt(None, "--concatenate", "-C", help="Concatenate output to a single file"),
|
||||
# fmt: on
|
||||
):
|
||||
"""
|
||||
|
||||
@@ -26,28 +26,10 @@ from ._util import (
|
||||
def debug_config_cli(
|
||||
# fmt: off
|
||||
ctx: typer.Context, # This is only used to read additional arguments
|
||||
config_path: Path = Arg(
|
||||
..., help="Path to config file", exists=True, allow_dash=True
|
||||
),
|
||||
code_path: Optional[Path] = Opt(
|
||||
None,
|
||||
"--code-path",
|
||||
"--code",
|
||||
"-c",
|
||||
help="Path to Python file with additional code (registered functions) to be imported",
|
||||
),
|
||||
show_funcs: bool = Opt(
|
||||
False,
|
||||
"--show-functions",
|
||||
"-F",
|
||||
help="Show an overview of all registered functions used in the config and where they come from (modules, files etc.)",
|
||||
),
|
||||
show_vars: bool = Opt(
|
||||
False,
|
||||
"--show-variables",
|
||||
"-V",
|
||||
help="Show an overview of all variables referenced in the config and their values. This will also reflect variables overwritten on the CLI.",
|
||||
),
|
||||
config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
|
||||
code_path: Optional[Path] = Opt(None, "--code-path", "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
|
||||
show_funcs: bool = Opt(False, "--show-functions", "-F", help="Show an overview of all registered functions used in the config and where they come from (modules, files etc.)"),
|
||||
show_vars: bool = Opt(False, "--show-variables", "-V", help="Show an overview of all variables referenced in the config and their values. This will also reflect variables overwritten on the CLI.")
|
||||
# fmt: on
|
||||
):
|
||||
"""Debug a config file and show validation errors. The command will
|
||||
@@ -82,10 +64,10 @@ def debug_config(
|
||||
config = nlp.config.interpolate()
|
||||
msg.divider("Config validation for [initialize]")
|
||||
with show_validation_error(config_path):
|
||||
T = registry.resolve(config["initialize"], schema=ConfigSchemaInit) # type: ignore[arg-type]
|
||||
T = registry.resolve(config["initialize"], schema=ConfigSchemaInit)
|
||||
msg.divider("Config validation for [training]")
|
||||
with show_validation_error(config_path):
|
||||
T = registry.resolve(config["training"], schema=ConfigSchemaTraining) # type: ignore[arg-type]
|
||||
T = registry.resolve(config["training"], schema=ConfigSchemaTraining)
|
||||
dot_names = [T["train_corpus"], T["dev_corpus"]]
|
||||
util.resolve_dot_names(config, dot_names)
|
||||
msg.good("Config is valid")
|
||||
|
||||
+12
-27
@@ -71,28 +71,11 @@ SPAN_LENGTH_THRESHOLD_PERCENTAGE = 90
|
||||
def debug_data_cli(
|
||||
# fmt: off
|
||||
ctx: typer.Context, # This is only used to read additional arguments
|
||||
config_path: Path = Arg(
|
||||
..., help="Path to config file", exists=True, allow_dash=True
|
||||
),
|
||||
code_path: Optional[Path] = Opt(
|
||||
None,
|
||||
"--code-path",
|
||||
"--code",
|
||||
"-c",
|
||||
help="Path to Python file with additional code (registered functions) to be imported",
|
||||
),
|
||||
ignore_warnings: bool = Opt(
|
||||
False,
|
||||
"--ignore-warnings",
|
||||
"-IW",
|
||||
help="Ignore warnings, only show stats and errors",
|
||||
),
|
||||
verbose: bool = Opt(
|
||||
False, "--verbose", "-V", help="Print additional information and explanations"
|
||||
),
|
||||
no_format: bool = Opt(
|
||||
False, "--no-format", "-NF", help="Don't pretty-print the results"
|
||||
),
|
||||
config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
|
||||
code_path: Optional[Path] = Opt(None, "--code-path", "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
|
||||
ignore_warnings: bool = Opt(False, "--ignore-warnings", "-IW", help="Ignore warnings, only show stats and errors"),
|
||||
verbose: bool = Opt(False, "--verbose", "-V", help="Print additional information and explanations"),
|
||||
no_format: bool = Opt(False, "--no-format", "-NF", help="Don't pretty-print the results"),
|
||||
# fmt: on
|
||||
):
|
||||
"""
|
||||
@@ -137,7 +120,7 @@ def debug_data(
|
||||
cfg = util.load_config(config_path, overrides=config_overrides)
|
||||
nlp = util.load_model_from_config(cfg)
|
||||
config = nlp.config.interpolate()
|
||||
T = registry.resolve(config["training"], schema=ConfigSchemaTraining) # type: ignore[arg-type]
|
||||
T = registry.resolve(config["training"], schema=ConfigSchemaTraining)
|
||||
# Use original config here, not resolved version
|
||||
sourced_components = get_sourced_components(cfg)
|
||||
frozen_components = T["frozen_components"]
|
||||
@@ -579,7 +562,7 @@ def debug_data(
|
||||
|
||||
if "morphologizer" in factory_names:
|
||||
msg.divider("Morphologizer (POS+Morph)")
|
||||
label_list = tuple(gold_train_data["morphs"])
|
||||
label_list = [label for label in gold_train_data["morphs"]]
|
||||
model_labels = _get_labels_from_model(nlp, "morphologizer")
|
||||
msg.info(f"{len(label_list)} label(s) in train data")
|
||||
labels = set(label_list)
|
||||
@@ -725,7 +708,7 @@ def debug_data(
|
||||
if len(dev_not_train) != 0:
|
||||
pct = len(dev_not_train) / len(trees_dev)
|
||||
msg.info(
|
||||
f"{len(dev_not_train)} lemmatizer trees ({pct * 100:.1f}% of dev trees)"
|
||||
f"{len(dev_not_train)} lemmatizer trees ({pct*100:.1f}% of dev trees)"
|
||||
" were found exclusively in the dev data."
|
||||
)
|
||||
else:
|
||||
@@ -985,14 +968,16 @@ def _compile_gold(
|
||||
|
||||
|
||||
@overload
|
||||
def _format_labels(labels: Iterable[str], counts: Literal[False] = False) -> str: ...
|
||||
def _format_labels(labels: Iterable[str], counts: Literal[False] = False) -> str:
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def _format_labels(
|
||||
labels: Iterable[Tuple[str, int]],
|
||||
counts: Literal[True],
|
||||
) -> str: ...
|
||||
) -> str:
|
||||
...
|
||||
|
||||
|
||||
def _format_labels(
|
||||
|
||||
+8
-31
@@ -2,10 +2,11 @@ from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import typer
|
||||
from thinc.api import Config
|
||||
from wasabi import MarkdownRenderer, Printer, diff_strings
|
||||
|
||||
from ..util import load_config
|
||||
from ._util import Arg, Opt, debug_cli, show_validation_error
|
||||
from ._util import Arg, Opt, debug_cli, parse_config_overrides, show_validation_error
|
||||
from .init_config import Optimizations, init_config
|
||||
|
||||
|
||||
@@ -16,36 +17,12 @@ from .init_config import Optimizations, init_config
|
||||
def debug_diff_cli(
|
||||
# fmt: off
|
||||
ctx: typer.Context,
|
||||
config_path: Path = Arg(
|
||||
..., help="Path to config file", exists=True, allow_dash=True
|
||||
),
|
||||
compare_to: Optional[Path] = Opt(
|
||||
None,
|
||||
help="Path to a config file to diff against, or `None` to compare against default settings",
|
||||
exists=True,
|
||||
allow_dash=True,
|
||||
),
|
||||
optimize: Optimizations = Opt(
|
||||
Optimizations.efficiency.value,
|
||||
"--optimize",
|
||||
"-o",
|
||||
help="Whether the user config was optimized for efficiency or accuracy. Only relevant when comparing against the default config.",
|
||||
),
|
||||
gpu: bool = Opt(
|
||||
False,
|
||||
"--gpu",
|
||||
"-G",
|
||||
help="Whether the original config can run on a GPU. Only relevant when comparing against the default config.",
|
||||
),
|
||||
pretraining: bool = Opt(
|
||||
False,
|
||||
"--pretraining",
|
||||
"--pt",
|
||||
help="Whether to compare on a config with pretraining involved. Only relevant when comparing against the default config.",
|
||||
),
|
||||
markdown: bool = Opt(
|
||||
False, "--markdown", "-md", help="Generate Markdown for GitHub issues"
|
||||
),
|
||||
config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
|
||||
compare_to: Optional[Path] = Opt(None, help="Path to a config file to diff against, or `None` to compare against default settings", exists=True, allow_dash=True),
|
||||
optimize: Optimizations = Opt(Optimizations.efficiency.value, "--optimize", "-o", help="Whether the user config was optimized for efficiency or accuracy. Only relevant when comparing against the default config."),
|
||||
gpu: bool = Opt(False, "--gpu", "-G", help="Whether the original config can run on a GPU. Only relevant when comparing against the default config."),
|
||||
pretraining: bool = Opt(False, "--pretraining", "--pt", help="Whether to compare on a config with pretraining involved. Only relevant when comparing against the default config."),
|
||||
markdown: bool = Opt(False, "--markdown", "-md", help="Generate Markdown for GitHub issues")
|
||||
# fmt: on
|
||||
):
|
||||
"""Show a diff of a config file with respect to spaCy's defaults or another config file. If
|
||||
|
||||
@@ -36,26 +36,18 @@ from ._util import (
|
||||
def debug_model_cli(
|
||||
# fmt: off
|
||||
ctx: typer.Context, # This is only used to read additional arguments
|
||||
config_path: Path = Arg(
|
||||
..., help="Path to config file", exists=True, allow_dash=True
|
||||
),
|
||||
component: str = Arg(
|
||||
..., help="Name of the pipeline component of which the model should be analysed"
|
||||
),
|
||||
layers: str = Opt(
|
||||
"", "--layers", "-l", help="Comma-separated names of layer IDs to print"
|
||||
),
|
||||
config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
|
||||
component: str = Arg(..., help="Name of the pipeline component of which the model should be analysed"),
|
||||
layers: str = Opt("", "--layers", "-l", help="Comma-separated names of layer IDs to print"),
|
||||
dimensions: bool = Opt(False, "--dimensions", "-DIM", help="Show dimensions"),
|
||||
parameters: bool = Opt(False, "--parameters", "-PAR", help="Show parameters"),
|
||||
gradients: bool = Opt(False, "--gradients", "-GRAD", help="Show gradients"),
|
||||
attributes: bool = Opt(False, "--attributes", "-ATTR", help="Show attributes"),
|
||||
P0: bool = Opt(False, "--print-step0", "-P0", help="Print model before training"),
|
||||
P1: bool = Opt(
|
||||
False, "--print-step1", "-P1", help="Print model after initialization"
|
||||
),
|
||||
P1: bool = Opt(False, "--print-step1", "-P1", help="Print model after initialization"),
|
||||
P2: bool = Opt(False, "--print-step2", "-P2", help="Print model after training"),
|
||||
P3: bool = Opt(False, "--print-step3", "-P3", help="Print final predictions"),
|
||||
use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
|
||||
use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU")
|
||||
# fmt: on
|
||||
):
|
||||
"""
|
||||
@@ -89,7 +81,7 @@ def debug_model_cli(
|
||||
with show_validation_error(config_path):
|
||||
nlp = util.load_model_from_config(raw_config)
|
||||
config = nlp.config.interpolate()
|
||||
T = registry.resolve(config["training"], schema=ConfigSchemaTraining) # type: ignore[arg-type]
|
||||
T = registry.resolve(config["training"], schema=ConfigSchemaTraining)
|
||||
seed = T["seed"]
|
||||
if seed is not None:
|
||||
msg.info(f"Fixing random seed: {seed}")
|
||||
@@ -178,7 +170,7 @@ def debug_model(
|
||||
msg.divider(f"STEP 3 - prediction")
|
||||
msg.info(str(prediction))
|
||||
|
||||
msg.good(f"Successfully ended analysis - model looks good.")
|
||||
msg.good(f"Succesfully ended analysis - model looks good.")
|
||||
|
||||
|
||||
def _sentences():
|
||||
|
||||
+7
-51
@@ -1,8 +1,5 @@
|
||||
import importlib.util
|
||||
import shutil
|
||||
import sys
|
||||
from typing import Optional, Sequence
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import requests
|
||||
import typer
|
||||
@@ -29,16 +26,8 @@ def download_cli(
|
||||
# fmt: off
|
||||
ctx: typer.Context,
|
||||
model: str = Arg(..., help="Name of pipeline package to download"),
|
||||
direct: bool = Opt(
|
||||
False, "--direct", "-d", "-D", help="Force direct download of name + version"
|
||||
),
|
||||
sdist: bool = Opt(
|
||||
False,
|
||||
"--sdist",
|
||||
"-S",
|
||||
help="Download sdist (.tar.gz) archive instead of pre-built binary wheel",
|
||||
),
|
||||
url: str = Opt(None, "--url", "-U", help="Download from given url"),
|
||||
direct: bool = Opt(False, "--direct", "-d", "-D", help="Force direct download of name + version"),
|
||||
sdist: bool = Opt(False, "--sdist", "-S", help="Download sdist (.tar.gz) archive instead of pre-built binary wheel"),
|
||||
# fmt: on
|
||||
):
|
||||
"""
|
||||
@@ -51,14 +40,13 @@ def download_cli(
|
||||
DOCS: https://spacy.io/api/cli#download
|
||||
AVAILABLE PACKAGES: https://spacy.io/models
|
||||
"""
|
||||
download(model, direct, sdist, url, *ctx.args)
|
||||
download(model, direct, sdist, *ctx.args)
|
||||
|
||||
|
||||
def download(
|
||||
model: str,
|
||||
direct: bool = False,
|
||||
sdist: bool = False,
|
||||
custom_url: Optional[str] = None,
|
||||
*pip_args,
|
||||
) -> None:
|
||||
if (
|
||||
@@ -75,13 +63,6 @@ def download(
|
||||
)
|
||||
pip_args = pip_args + ("--no-deps",)
|
||||
if direct:
|
||||
# Reject model names with '/', in order to prevent shenanigans.
|
||||
if "/" in model:
|
||||
msg.fail(
|
||||
title="Model download rejected",
|
||||
text=f"Cannot download model '{model}'. Models are expected to be file names, not URLs or fragments",
|
||||
exits=True,
|
||||
)
|
||||
components = model.split("-")
|
||||
model_name = "".join(components[:-1])
|
||||
version = components[-1]
|
||||
@@ -98,7 +79,7 @@ def download(
|
||||
|
||||
filename = get_model_filename(model_name, version, sdist)
|
||||
|
||||
download_model(filename, pip_args, custom_url)
|
||||
download_model(filename, pip_args)
|
||||
msg.good(
|
||||
"Download and installation successful",
|
||||
f"You can now load the package via spacy.load('{model_name}')",
|
||||
@@ -170,34 +151,9 @@ def get_latest_version(model: str) -> str:
|
||||
|
||||
|
||||
def download_model(
|
||||
filename: str,
|
||||
user_pip_args: Optional[Sequence[str]] = None,
|
||||
custom_url: Optional[str] = None,
|
||||
filename: str, user_pip_args: Optional[Sequence[str]] = None
|
||||
) -> None:
|
||||
# Construct the download URL carefully. We need to make sure we don't
|
||||
# allow relative paths or other shenanigans to trick us into download
|
||||
# from outside our own repo.
|
||||
base_url = custom_url if custom_url else about.__download_url__
|
||||
# urljoin requires that the path ends with /, or the last path part will be dropped
|
||||
if not base_url.endswith("/"):
|
||||
base_url = about.__download_url__ + "/"
|
||||
download_url = urljoin(base_url, filename)
|
||||
if not download_url.startswith(about.__download_url__):
|
||||
raise ValueError(f"Download from {filename} rejected. Was it a relative path?")
|
||||
download_url = about.__download_url__ + "/" + filename
|
||||
pip_args = list(user_pip_args) if user_pip_args is not None else []
|
||||
cmd = _get_pip_install_cmd() + pip_args + [download_url]
|
||||
cmd = [sys.executable, "-m", "pip", "install"] + pip_args + [download_url]
|
||||
run_command(cmd)
|
||||
|
||||
|
||||
def _get_pip_install_cmd() -> list:
|
||||
if importlib.util.find_spec("pip") is not None:
|
||||
return [sys.executable, "-m", "pip", "install"]
|
||||
elif shutil.which("uv"):
|
||||
return ["uv", "pip", "install"]
|
||||
else:
|
||||
msg.fail(
|
||||
"No package installer found",
|
||||
"spaCy requires either pip or uv to download models. "
|
||||
"Please install one of them and try again.",
|
||||
exits=1,
|
||||
)
|
||||
|
||||
+11
-37
@@ -1,12 +1,13 @@
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import srsly
|
||||
from thinc.api import fix_random_seed
|
||||
from wasabi import Printer
|
||||
|
||||
from .. import displacy, util
|
||||
from ..scorer import Scorer
|
||||
from ..tokens import Doc
|
||||
from ..training import Corpus
|
||||
from ._util import Arg, Opt, app, benchmark_cli, import_code, setup_gpu
|
||||
@@ -19,42 +20,15 @@ from ._util import Arg, Opt, app, benchmark_cli, import_code, setup_gpu
|
||||
def evaluate_cli(
|
||||
# fmt: off
|
||||
model: str = Arg(..., help="Model name or path"),
|
||||
data_path: Path = Arg(
|
||||
..., help="Location of binary evaluation data in .spacy format", exists=True
|
||||
),
|
||||
output: Optional[Path] = Opt(
|
||||
None, "--output", "-o", help="Output JSON file for metrics", dir_okay=False
|
||||
),
|
||||
code_path: Optional[Path] = Opt(
|
||||
None,
|
||||
"--code",
|
||||
"-c",
|
||||
help="Path to Python file with additional code (registered functions) to be imported",
|
||||
),
|
||||
data_path: Path = Arg(..., help="Location of binary evaluation data in .spacy format", exists=True),
|
||||
output: Optional[Path] = Opt(None, "--output", "-o", help="Output JSON file for metrics", dir_okay=False),
|
||||
code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
|
||||
use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
|
||||
gold_preproc: bool = Opt(
|
||||
False, "--gold-preproc", "-G", help="Use gold preprocessing"
|
||||
),
|
||||
displacy_path: Optional[Path] = Opt(
|
||||
None,
|
||||
"--displacy-path",
|
||||
"-dp",
|
||||
help="Directory to output rendered parses as HTML",
|
||||
exists=True,
|
||||
file_okay=False,
|
||||
),
|
||||
displacy_limit: int = Opt(
|
||||
25, "--displacy-limit", "-dl", help="Limit of parses to render as HTML"
|
||||
),
|
||||
per_component: bool = Opt(
|
||||
False,
|
||||
"--per-component",
|
||||
"-P",
|
||||
help="Return scores per component, only applicable when an output JSON file is specified.",
|
||||
),
|
||||
spans_key: str = Opt(
|
||||
"sc", "--spans-key", "-sk", help="Spans key to use when evaluating Doc.spans"
|
||||
),
|
||||
gold_preproc: bool = Opt(False, "--gold-preproc", "-G", help="Use gold preprocessing"),
|
||||
displacy_path: Optional[Path] = Opt(None, "--displacy-path", "-dp", help="Directory to output rendered parses as HTML", exists=True, file_okay=False),
|
||||
displacy_limit: int = Opt(25, "--displacy-limit", "-dl", help="Limit of parses to render as HTML"),
|
||||
per_component: bool = Opt(False, "--per-component", "-P", help="Return scores per component, only applicable when an output JSON file is specified."),
|
||||
spans_key: str = Opt("sc", "--spans-key", "-sk", help="Spans key to use when evaluating Doc.spans"),
|
||||
# fmt: on
|
||||
):
|
||||
"""
|
||||
@@ -149,7 +123,7 @@ def evaluate(
|
||||
if key == "speed":
|
||||
results[metric] = f"{scores[key]:.0f}"
|
||||
else:
|
||||
results[metric] = f"{scores[key] * 100:.2f}"
|
||||
results[metric] = f"{scores[key]*100:.2f}"
|
||||
else:
|
||||
results[metric] = "-"
|
||||
data[re.sub(r"[\s/]", "_", key.lower())] = scores[key]
|
||||
|
||||
@@ -11,9 +11,7 @@ from ._util import Arg, Opt, app
|
||||
def find_function_cli(
|
||||
# fmt: off
|
||||
func_name: str = Arg(..., help="Name of the registered function."),
|
||||
registry_name: Optional[str] = Opt(
|
||||
None, "--registry", "-r", help="Name of the catalogue registry."
|
||||
),
|
||||
registry_name: Optional[str] = Opt(None, "--registry", "-r", help="Name of the catalogue registry."),
|
||||
# fmt: on
|
||||
):
|
||||
"""
|
||||
|
||||
+22
-50
@@ -27,43 +27,19 @@ _DEFAULTS = {
|
||||
def find_threshold_cli(
|
||||
# fmt: off
|
||||
model: str = Arg(..., help="Model name or path"),
|
||||
data_path: Path = Arg(
|
||||
..., help="Location of binary evaluation data in .spacy format", exists=True
|
||||
),
|
||||
data_path: Path = Arg(..., help="Location of binary evaluation data in .spacy format", exists=True),
|
||||
pipe_name: str = Arg(..., help="Name of pipe to examine thresholds for"),
|
||||
threshold_key: str = Arg(
|
||||
..., help="Key of threshold attribute in component's configuration"
|
||||
),
|
||||
threshold_key: str = Arg(..., help="Key of threshold attribute in component's configuration"),
|
||||
scores_key: str = Arg(..., help="Metric to optimize"),
|
||||
n_trials: int = Opt(
|
||||
_DEFAULTS["n_trials"],
|
||||
"--n_trials",
|
||||
"-n",
|
||||
help="Number of trials to determine optimal thresholds",
|
||||
),
|
||||
code_path: Optional[Path] = Opt(
|
||||
None,
|
||||
"--code",
|
||||
"-c",
|
||||
help="Path to Python file with additional code (registered functions) to be imported",
|
||||
),
|
||||
use_gpu: int = Opt(
|
||||
_DEFAULTS["use_gpu"], "--gpu-id", "-g", help="GPU ID or -1 for CPU"
|
||||
),
|
||||
gold_preproc: bool = Opt(
|
||||
_DEFAULTS["gold_preproc"], "--gold-preproc", "-G", help="Use gold preprocessing"
|
||||
),
|
||||
verbose: bool = Opt(
|
||||
False,
|
||||
"--verbose",
|
||||
"-V",
|
||||
"-VV",
|
||||
help="Display more information for debugging purposes",
|
||||
),
|
||||
n_trials: int = Opt(_DEFAULTS["n_trials"], "--n_trials", "-n", help="Number of trials to determine optimal thresholds"),
|
||||
code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
|
||||
use_gpu: int = Opt(_DEFAULTS["use_gpu"], "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
|
||||
gold_preproc: bool = Opt(_DEFAULTS["gold_preproc"], "--gold-preproc", "-G", help="Use gold preprocessing"),
|
||||
verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
|
||||
# fmt: on
|
||||
):
|
||||
"""
|
||||
Runs prediction trials for a trained model with varying thresholds to maximize
|
||||
Runs prediction trials for a trained model with varying tresholds to maximize
|
||||
the specified metric. The search space for the threshold is traversed linearly
|
||||
from 0 to 1 in `n_trials` steps. Results are displayed in a table on `stdout`
|
||||
(the corresponding API call to `spacy.cli.find_threshold.find_threshold()`
|
||||
@@ -105,7 +81,7 @@ def find_threshold(
|
||||
silent: bool = True,
|
||||
) -> Tuple[float, float, Dict[float, float]]:
|
||||
"""
|
||||
Runs prediction trials for models with varying thresholds to maximize the specified metric.
|
||||
Runs prediction trials for models with varying tresholds to maximize the specified metric.
|
||||
model (Union[str, Path]): Pipeline to evaluate. Can be a package or a path to a data directory.
|
||||
data_path (Path): Path to file with DocBin with docs to use for threshold search.
|
||||
pipe_name (str): Name of pipe to examine thresholds for.
|
||||
@@ -181,11 +157,9 @@ def find_threshold(
|
||||
exits=1,
|
||||
)
|
||||
return {
|
||||
keys[0]: (
|
||||
filter_config(config[keys[0]], keys[1:], full_key)
|
||||
if len(keys) > 1
|
||||
else config[keys[0]]
|
||||
)
|
||||
keys[0]: filter_config(config[keys[0]], keys[1:], full_key)
|
||||
if len(keys) > 1
|
||||
else config[keys[0]]
|
||||
}
|
||||
|
||||
# Evaluate with varying threshold values.
|
||||
@@ -207,10 +181,10 @@ def find_threshold(
|
||||
),
|
||||
)
|
||||
if hasattr(pipe, "cfg"):
|
||||
nlp.get_pipe(pipe_name).cfg = set_nested_item( # type: ignore[attr-defined]
|
||||
pipe.cfg,
|
||||
config_keys,
|
||||
threshold, # type: ignore[attr-defined]
|
||||
setattr(
|
||||
nlp.get_pipe(pipe_name),
|
||||
"cfg",
|
||||
set_nested_item(getattr(pipe, "cfg"), config_keys, threshold),
|
||||
)
|
||||
|
||||
eval_scores = nlp.evaluate(dev_dataset)
|
||||
@@ -242,14 +216,12 @@ def find_threshold(
|
||||
if len(set(scores.values())) == 1:
|
||||
wasabi.msg.warn(
|
||||
title="All scores are identical. Verify that all settings are correct.",
|
||||
text=(
|
||||
""
|
||||
if (
|
||||
not isinstance(pipe, MultiLabel_TextCategorizer)
|
||||
or scores_key in ("cats_macro_f", "cats_micro_f")
|
||||
)
|
||||
else "Use `cats_macro_f` or `cats_micro_f` when optimizing the threshold for `textcat_multilabel`."
|
||||
),
|
||||
text=""
|
||||
if (
|
||||
not isinstance(pipe, MultiLabel_TextCategorizer)
|
||||
or scores_key in ("cats_macro_f", "cats_micro_f")
|
||||
)
|
||||
else "Use `cats_macro_f` or `cats_micro_f` when optimizing the threshold for `textcat_multilabel`.",
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
+4
-18
@@ -16,24 +16,10 @@ from .download import get_latest_version, get_model_filename
|
||||
def info_cli(
|
||||
# fmt: off
|
||||
model: Optional[str] = Arg(None, help="Optional loadable spaCy pipeline"),
|
||||
markdown: bool = Opt(
|
||||
False, "--markdown", "-md", help="Generate Markdown for GitHub issues"
|
||||
),
|
||||
silent: bool = Opt(
|
||||
False, "--silent", "-s", "-S", help="Don't print anything (just return)"
|
||||
),
|
||||
exclude: str = Opt(
|
||||
"labels",
|
||||
"--exclude",
|
||||
"-e",
|
||||
help="Comma-separated keys to exclude from the print-out",
|
||||
),
|
||||
url: bool = Opt(
|
||||
False,
|
||||
"--url",
|
||||
"-u",
|
||||
help="Print the URL to download the most recent compatible version of the pipeline",
|
||||
),
|
||||
markdown: bool = Opt(False, "--markdown", "-md", help="Generate Markdown for GitHub issues"),
|
||||
silent: bool = Opt(False, "--silent", "-s", "-S", help="Don't print anything (just return)"),
|
||||
exclude: str = Opt("labels", "--exclude", "-e", help="Comma-separated keys to exclude from the print-out"),
|
||||
url: bool = Opt(False, "--url", "-u", help="Print the URL to download the most recent compatible version of the pipeline"),
|
||||
# fmt: on
|
||||
):
|
||||
"""
|
||||
|
||||
+16
-66
@@ -49,44 +49,13 @@ class InitValues:
|
||||
@init_cli.command("config")
|
||||
def init_config_cli(
|
||||
# fmt: off
|
||||
output_file: Path = Arg(
|
||||
...,
|
||||
help="File to save the config to or - for stdout (will only output config and no additional logging info)",
|
||||
allow_dash=True,
|
||||
),
|
||||
lang: str = Opt(
|
||||
InitValues.lang, "--lang", "-l", help="Two-letter code of the language to use"
|
||||
),
|
||||
pipeline: str = Opt(
|
||||
",".join(InitValues.pipeline),
|
||||
"--pipeline",
|
||||
"-p",
|
||||
help="Comma-separated names of trainable pipeline components to include (without 'tok2vec' or 'transformer')",
|
||||
),
|
||||
optimize: Optimizations = Opt(
|
||||
InitValues.optimize,
|
||||
"--optimize",
|
||||
"-o",
|
||||
help="Whether to optimize for efficiency (faster inference, smaller model, lower memory consumption) or higher accuracy (potentially larger and slower model). This will impact the choice of architecture, pretrained weights and related hyperparameters.",
|
||||
),
|
||||
gpu: bool = Opt(
|
||||
InitValues.gpu,
|
||||
"--gpu",
|
||||
"-G",
|
||||
help="Whether the model can run on GPU. This will impact the choice of architecture, pretrained weights and related hyperparameters.",
|
||||
),
|
||||
pretraining: bool = Opt(
|
||||
InitValues.pretraining,
|
||||
"--pretraining",
|
||||
"-pt",
|
||||
help="Include config for pretraining (with 'spacy pretrain')",
|
||||
),
|
||||
force_overwrite: bool = Opt(
|
||||
InitValues.force_overwrite,
|
||||
"--force",
|
||||
"-F",
|
||||
help="Force overwriting the output file",
|
||||
),
|
||||
output_file: Path = Arg(..., help="File to save the config to or - for stdout (will only output config and no additional logging info)", allow_dash=True),
|
||||
lang: str = Opt(InitValues.lang, "--lang", "-l", help="Two-letter code of the language to use"),
|
||||
pipeline: str = Opt(",".join(InitValues.pipeline), "--pipeline", "-p", help="Comma-separated names of trainable pipeline components to include (without 'tok2vec' or 'transformer')"),
|
||||
optimize: Optimizations = Opt(InitValues.optimize, "--optimize", "-o", help="Whether to optimize for efficiency (faster inference, smaller model, lower memory consumption) or higher accuracy (potentially larger and slower model). This will impact the choice of architecture, pretrained weights and related hyperparameters."),
|
||||
gpu: bool = Opt(InitValues.gpu, "--gpu", "-G", help="Whether the model can run on GPU. This will impact the choice of architecture, pretrained weights and related hyperparameters."),
|
||||
pretraining: bool = Opt(InitValues.pretraining, "--pretraining", "-pt", help="Include config for pretraining (with 'spacy pretrain')"),
|
||||
force_overwrite: bool = Opt(InitValues.force_overwrite, "--force", "-F", help="Force overwriting the output file"),
|
||||
# fmt: on
|
||||
):
|
||||
"""
|
||||
@@ -119,28 +88,11 @@ def init_config_cli(
|
||||
@init_cli.command("fill-config")
|
||||
def init_fill_config_cli(
|
||||
# fmt: off
|
||||
base_path: Path = Arg(
|
||||
..., help="Path to base config to fill", exists=True, dir_okay=False
|
||||
),
|
||||
output_file: Path = Arg(
|
||||
"-", help="Path to output .cfg file (or - for stdout)", allow_dash=True
|
||||
),
|
||||
pretraining: bool = Opt(
|
||||
False,
|
||||
"--pretraining",
|
||||
"-pt",
|
||||
help="Include config for pretraining (with 'spacy pretrain')",
|
||||
),
|
||||
diff: bool = Opt(
|
||||
False, "--diff", "-D", help="Print a visual diff highlighting the changes"
|
||||
),
|
||||
code_path: Optional[Path] = Opt(
|
||||
None,
|
||||
"--code-path",
|
||||
"--code",
|
||||
"-c",
|
||||
help="Path to Python file with additional code (registered functions) to be imported",
|
||||
),
|
||||
base_path: Path = Arg(..., help="Path to base config to fill", exists=True, dir_okay=False),
|
||||
output_file: Path = Arg("-", help="Path to output .cfg file (or - for stdout)", allow_dash=True),
|
||||
pretraining: bool = Opt(False, "--pretraining", "-pt", help="Include config for pretraining (with 'spacy pretrain')"),
|
||||
diff: bool = Opt(False, "--diff", "-D", help="Print a visual diff highlighting the changes"),
|
||||
code_path: Optional[Path] = Opt(None, "--code-path", "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
|
||||
# fmt: on
|
||||
):
|
||||
"""
|
||||
@@ -216,7 +168,7 @@ def init_config(
|
||||
# Filter out duplicates since tok2vec and transformer are added by template
|
||||
pipeline = [pipe for pipe in pipeline if pipe not in ("tok2vec", "transformer")]
|
||||
defaults = RECOMMENDATIONS["__default__"]
|
||||
reco = RecommendationSchema(**RECOMMENDATIONS.get(lang, defaults)).model_dump()
|
||||
reco = RecommendationSchema(**RECOMMENDATIONS.get(lang, defaults)).dict()
|
||||
variables = {
|
||||
"lang": lang,
|
||||
"components": pipeline,
|
||||
@@ -243,11 +195,9 @@ def init_config(
|
||||
"Pipeline": ", ".join(pipeline),
|
||||
"Optimize for": optimize,
|
||||
"Hardware": variables["hardware"].upper(),
|
||||
"Transformer": (
|
||||
template_vars.transformer.get("name") # type: ignore[attr-defined]
|
||||
if template_vars.use_transformer # type: ignore[attr-defined]
|
||||
else None
|
||||
),
|
||||
"Transformer": template_vars.transformer.get("name") # type: ignore[attr-defined]
|
||||
if template_vars.use_transformer # type: ignore[attr-defined]
|
||||
else None,
|
||||
}
|
||||
msg.info("Generated config template specific for your use case")
|
||||
for label, value in use_case.items():
|
||||
|
||||
+14
-69
@@ -26,42 +26,13 @@ def init_vectors_cli(
|
||||
lang: str = Arg(..., help="The language of the nlp object to create"),
|
||||
vectors_loc: Path = Arg(..., help="Vectors file in Word2Vec format", exists=True),
|
||||
output_dir: Path = Arg(..., help="Pipeline output directory"),
|
||||
prune: int = Opt(
|
||||
-1, "--prune", "-p", help="Optional number of vectors to prune to"
|
||||
),
|
||||
truncate: int = Opt(
|
||||
0,
|
||||
"--truncate",
|
||||
"-t",
|
||||
help="Optional number of vectors to truncate to when reading in vectors file",
|
||||
),
|
||||
prune: int = Opt(-1, "--prune", "-p", help="Optional number of vectors to prune to"),
|
||||
truncate: int = Opt(0, "--truncate", "-t", help="Optional number of vectors to truncate to when reading in vectors file"),
|
||||
mode: str = Opt("default", "--mode", "-m", help="Vectors mode: default or floret"),
|
||||
name: Optional[str] = Opt(
|
||||
None,
|
||||
"--name",
|
||||
"-n",
|
||||
help="Optional name for the word vectors, e.g. en_core_web_lg.vectors",
|
||||
),
|
||||
verbose: bool = Opt(
|
||||
False,
|
||||
"--verbose",
|
||||
"-V",
|
||||
"-VV",
|
||||
help="Display more information for debugging purposes",
|
||||
),
|
||||
jsonl_loc: Optional[Path] = Opt(
|
||||
None,
|
||||
"--lexemes-jsonl",
|
||||
"-j",
|
||||
help="Location of JSONL-formatted attributes file",
|
||||
hidden=True,
|
||||
),
|
||||
attr: str = Opt(
|
||||
"ORTH",
|
||||
"--attr",
|
||||
"-a",
|
||||
help="Optional token attribute to use for vectors, e.g. LOWER or NORM",
|
||||
),
|
||||
name: Optional[str] = Opt(None, "--name", "-n", help="Optional name for the word vectors, e.g. en_core_web_lg.vectors"),
|
||||
verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
|
||||
jsonl_loc: Optional[Path] = Opt(None, "--lexemes-jsonl", "-j", help="Location of JSONL-formatted attributes file", hidden=True),
|
||||
attr: str = Opt("ORTH", "--attr", "-a", help="Optional token attribute to use for vectors, e.g. LOWER or NORM"),
|
||||
# fmt: on
|
||||
):
|
||||
"""Convert word vectors for use with spaCy. Will export an nlp object that
|
||||
@@ -110,24 +81,11 @@ def update_lexemes(nlp: Language, jsonl_loc: Path) -> None:
|
||||
def init_pipeline_cli(
|
||||
# fmt: off
|
||||
ctx: typer.Context, # This is only used to read additional arguments
|
||||
config_path: Path = Arg(
|
||||
..., help="Path to config file", exists=True, allow_dash=True
|
||||
),
|
||||
config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
|
||||
output_path: Path = Arg(..., help="Output directory for the prepared data"),
|
||||
code_path: Optional[Path] = Opt(
|
||||
None,
|
||||
"--code",
|
||||
"-c",
|
||||
help="Path to Python file with additional code (registered functions) to be imported",
|
||||
),
|
||||
verbose: bool = Opt(
|
||||
False,
|
||||
"--verbose",
|
||||
"-V",
|
||||
"-VV",
|
||||
help="Display more information for debugging purposes",
|
||||
),
|
||||
use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
|
||||
code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
|
||||
verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
|
||||
use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU")
|
||||
# fmt: on
|
||||
):
|
||||
if verbose:
|
||||
@@ -150,24 +108,11 @@ def init_pipeline_cli(
|
||||
def init_labels_cli(
|
||||
# fmt: off
|
||||
ctx: typer.Context, # This is only used to read additional arguments
|
||||
config_path: Path = Arg(
|
||||
..., help="Path to config file", exists=True, allow_dash=True
|
||||
),
|
||||
config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
|
||||
output_path: Path = Arg(..., help="Output directory for the labels"),
|
||||
code_path: Optional[Path] = Opt(
|
||||
None,
|
||||
"--code",
|
||||
"-c",
|
||||
help="Path to Python file with additional code (registered functions) to be imported",
|
||||
),
|
||||
verbose: bool = Opt(
|
||||
False,
|
||||
"--verbose",
|
||||
"-V",
|
||||
"-VV",
|
||||
help="Display more information for debugging purposes",
|
||||
),
|
||||
use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
|
||||
code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
|
||||
verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
|
||||
use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU")
|
||||
# fmt: on
|
||||
):
|
||||
"""Generate JSON files for the labels in the data. This helps speed up the
|
||||
|
||||
+18
-69
@@ -21,56 +21,15 @@ from ._util import SDIST_SUFFIX, WHEEL_SUFFIX, Arg, Opt, app, string_to_list
|
||||
@app.command("package")
|
||||
def package_cli(
|
||||
# fmt: off
|
||||
input_dir: Path = Arg(
|
||||
..., help="Directory with pipeline data", exists=True, file_okay=False
|
||||
),
|
||||
output_dir: Path = Arg(
|
||||
..., help="Output parent directory", exists=True, file_okay=False
|
||||
),
|
||||
code_paths: str = Opt(
|
||||
"",
|
||||
"--code",
|
||||
"-c",
|
||||
help="Comma-separated paths to Python file with additional code (registered functions) to be included in the package",
|
||||
),
|
||||
meta_path: Optional[Path] = Opt(
|
||||
None,
|
||||
"--meta-path",
|
||||
"--meta",
|
||||
"-m",
|
||||
help="Path to meta.json",
|
||||
exists=True,
|
||||
dir_okay=False,
|
||||
),
|
||||
create_meta: bool = Opt(
|
||||
False, "--create-meta", "-C", help="Create meta.json, even if one exists"
|
||||
),
|
||||
name: Optional[str] = Opt(
|
||||
None, "--name", "-n", help="Package name to override meta"
|
||||
),
|
||||
version: Optional[str] = Opt(
|
||||
None, "--version", "-v", help="Package version to override meta"
|
||||
),
|
||||
build: str = Opt(
|
||||
"sdist",
|
||||
"--build",
|
||||
"-b",
|
||||
help="Comma-separated formats to build: sdist and/or wheel, or none.",
|
||||
),
|
||||
force: bool = Opt(
|
||||
False,
|
||||
"--force",
|
||||
"-f",
|
||||
"-F",
|
||||
help="Force overwriting existing data in output directory",
|
||||
),
|
||||
require_parent: bool = Opt(
|
||||
True,
|
||||
"--require-parent/--no-require-parent",
|
||||
"-R",
|
||||
"-R",
|
||||
help="Include the parent package (e.g. spacy) in the requirements",
|
||||
),
|
||||
input_dir: Path = Arg(..., help="Directory with pipeline data", exists=True, file_okay=False),
|
||||
output_dir: Path = Arg(..., help="Output parent directory", exists=True, file_okay=False),
|
||||
code_paths: str = Opt("", "--code", "-c", help="Comma-separated paths to Python file with additional code (registered functions) to be included in the package"),
|
||||
meta_path: Optional[Path] = Opt(None, "--meta-path", "--meta", "-m", help="Path to meta.json", exists=True, dir_okay=False),
|
||||
create_meta: bool = Opt(False, "--create-meta", "-C", help="Create meta.json, even if one exists"),
|
||||
name: Optional[str] = Opt(None, "--name", "-n", help="Package name to override meta"),
|
||||
version: Optional[str] = Opt(None, "--version", "-v", help="Package version to override meta"),
|
||||
build: str = Opt("sdist", "--build", "-b", help="Comma-separated formats to build: sdist and/or wheel, or none."),
|
||||
force: bool = Opt(False, "--force", "-f", "-F", help="Force overwriting existing data in output directory"),
|
||||
# fmt: on
|
||||
):
|
||||
"""
|
||||
@@ -101,7 +60,6 @@ def package_cli(
|
||||
create_sdist=create_sdist,
|
||||
create_wheel=create_wheel,
|
||||
force=force,
|
||||
require_parent=require_parent,
|
||||
silent=False,
|
||||
)
|
||||
|
||||
@@ -116,7 +74,6 @@ def package(
|
||||
create_meta: bool = False,
|
||||
create_sdist: bool = True,
|
||||
create_wheel: bool = False,
|
||||
require_parent: bool = False,
|
||||
force: bool = False,
|
||||
silent: bool = True,
|
||||
) -> None:
|
||||
@@ -156,7 +113,7 @@ def package(
|
||||
if not meta_path.exists() or not meta_path.is_file():
|
||||
msg.fail("Can't load pipeline meta.json", meta_path, exits=1)
|
||||
meta = srsly.read_json(meta_path)
|
||||
meta = get_meta(input_dir, meta, require_parent=require_parent)
|
||||
meta = get_meta(input_dir, meta)
|
||||
if meta["requirements"]:
|
||||
msg.good(
|
||||
f"Including {len(meta['requirements'])} package requirement(s) from "
|
||||
@@ -229,7 +186,6 @@ def package(
|
||||
imports.append(code_path.stem)
|
||||
shutil.copy(str(code_path), str(package_path))
|
||||
create_file(main_path / "meta.json", srsly.json_dumps(meta, indent=2))
|
||||
|
||||
create_file(main_path / "setup.py", TEMPLATE_SETUP)
|
||||
create_file(main_path / "MANIFEST.in", TEMPLATE_MANIFEST)
|
||||
init_py = TEMPLATE_INIT.format(
|
||||
@@ -346,8 +302,6 @@ def get_third_party_dependencies(
|
||||
modules.add(func_info["module"].split(".")[0]) # type: ignore[union-attr]
|
||||
dependencies = []
|
||||
for module_name in modules:
|
||||
if module_name == about.__title__:
|
||||
continue
|
||||
if module_name in distributions:
|
||||
dist = distributions.get(module_name)
|
||||
if dist:
|
||||
@@ -378,9 +332,7 @@ def create_file(file_path: Path, contents: str) -> None:
|
||||
|
||||
|
||||
def get_meta(
|
||||
model_path: Union[str, Path],
|
||||
existing_meta: Dict[str, Any],
|
||||
require_parent: bool = False,
|
||||
model_path: Union[str, Path], existing_meta: Dict[str, Any]
|
||||
) -> Dict[str, Any]:
|
||||
meta: Dict[str, Any] = {
|
||||
"lang": "en",
|
||||
@@ -409,8 +361,6 @@ def get_meta(
|
||||
existing_reqs = [util.split_requirement(req)[0] for req in meta["requirements"]]
|
||||
reqs = get_third_party_dependencies(nlp.config, exclude=existing_reqs)
|
||||
meta["requirements"].extend(reqs)
|
||||
if require_parent and about.__title__ not in meta["requirements"]:
|
||||
meta["requirements"].append(about.__title__ + meta["spacy_version"])
|
||||
return meta
|
||||
|
||||
|
||||
@@ -450,7 +400,7 @@ def generate_readme(meta: Dict[str, Any]) -> str:
|
||||
pipeline = ", ".join([md.code(p) for p in meta.get("pipeline", [])])
|
||||
components = ", ".join([md.code(p) for p in meta.get("components", [])])
|
||||
vecs = meta.get("vectors", {})
|
||||
vectors = f"{vecs.get('keys', 0)} keys, {vecs.get('vectors', 0)} unique vectors ({vecs.get('width', 0)} dimensions)"
|
||||
vectors = f"{vecs.get('keys', 0)} keys, {vecs.get('vectors', 0)} unique vectors ({ vecs.get('width', 0)} dimensions)"
|
||||
author = meta.get("author") or "n/a"
|
||||
notes = meta.get("notes", "")
|
||||
license_name = meta.get("license")
|
||||
@@ -509,7 +459,7 @@ def _format_accuracy(data: Dict[str, Any], exclude: List[str] = ["speed"]) -> st
|
||||
md = MarkdownRenderer()
|
||||
scalars = [(k, v) for k, v in data.items() if isinstance(v, (int, float))]
|
||||
scores = [
|
||||
(md.code(acc.upper()), f"{score * 100:.2f}")
|
||||
(md.code(acc.upper()), f"{score*100:.2f}")
|
||||
for acc, score in scalars
|
||||
if acc not in exclude
|
||||
]
|
||||
@@ -528,7 +478,9 @@ def _format_label_scheme(data: Dict[str, Any]) -> str:
|
||||
if not labels:
|
||||
continue
|
||||
col1 = md.bold(md.code(pipe))
|
||||
col2 = ", ".join([md.code(str(label).replace("|", "\\|")) for label in labels]) # noqa: W605
|
||||
col2 = ", ".join(
|
||||
[md.code(str(label).replace("|", "\\|")) for label in labels]
|
||||
) # noqa: W605
|
||||
label_data.append((col1, col2))
|
||||
n_labels += len(labels)
|
||||
n_pipes += 1
|
||||
@@ -583,11 +535,8 @@ def list_files(data_dir):
|
||||
|
||||
|
||||
def list_requirements(meta):
|
||||
# Up to version 3.7, we included the parent package
|
||||
# in requirements by default. This behaviour is removed
|
||||
# in 3.8, with a setting to include the parent package in
|
||||
# the requirements list in the meta if desired.
|
||||
requirements = []
|
||||
parent_package = meta.get('parent_package', 'spacy')
|
||||
requirements = [parent_package + meta['spacy_version']]
|
||||
if 'setup_requires' in meta:
|
||||
requirements += meta['setup_requires']
|
||||
if 'requirements' in meta:
|
||||
|
||||
+5
-24
@@ -25,32 +25,13 @@ from ._util import (
|
||||
def pretrain_cli(
|
||||
# fmt: off
|
||||
ctx: typer.Context, # This is only used to read additional arguments
|
||||
config_path: Path = Arg(
|
||||
..., help="Path to config file", exists=True, dir_okay=False, allow_dash=True
|
||||
),
|
||||
config_path: Path = Arg(..., help="Path to config file", exists=True, dir_okay=False, allow_dash=True),
|
||||
output_dir: Path = Arg(..., help="Directory to write weights to on each epoch"),
|
||||
code_path: Optional[Path] = Opt(
|
||||
None,
|
||||
"--code",
|
||||
"-c",
|
||||
help="Path to Python file with additional code (registered functions) to be imported",
|
||||
),
|
||||
resume_path: Optional[Path] = Opt(
|
||||
None,
|
||||
"--resume-path",
|
||||
"-r",
|
||||
help="Path to pretrained weights from which to resume pretraining",
|
||||
),
|
||||
epoch_resume: Optional[int] = Opt(
|
||||
None,
|
||||
"--epoch-resume",
|
||||
"-er",
|
||||
help="The epoch to resume counting from when using --resume-path. Prevents unintended overwriting of existing weight files.",
|
||||
),
|
||||
code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
|
||||
resume_path: Optional[Path] = Opt(None, "--resume-path", "-r", help="Path to pretrained weights from which to resume pretraining"),
|
||||
epoch_resume: Optional[int] = Opt(None, "--epoch-resume", "-er", help="The epoch to resume counting from when using --resume-path. Prevents unintended overwriting of existing weight files."),
|
||||
use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
|
||||
skip_last: bool = Opt(
|
||||
False, "--skip-last", "-L", help="Skip saving model-last.bin"
|
||||
),
|
||||
skip_last: bool = Opt(False, "--skip-last", "-L", help="Skip saving model-last.bin"),
|
||||
# fmt: on
|
||||
):
|
||||
"""
|
||||
|
||||
+3
-10
@@ -21,15 +21,8 @@ def profile_cli(
|
||||
# fmt: off
|
||||
ctx: typer.Context, # This is only used to read current calling context
|
||||
model: str = Arg(..., help="Trained pipeline to load"),
|
||||
inputs: Optional[Path] = Arg(
|
||||
None,
|
||||
help="Location of input file. '-' for stdin.",
|
||||
exists=True,
|
||||
allow_dash=True,
|
||||
),
|
||||
n_texts: int = Opt(
|
||||
10000, "--n-texts", "-n", help="Maximum number of texts to use if available"
|
||||
),
|
||||
inputs: Optional[Path] = Arg(None, help="Location of input file. '-' for stdin.", exists=True, allow_dash=True),
|
||||
n_texts: int = Opt(10000, "--n-texts", "-n", help="Maximum number of texts to use if available"),
|
||||
# fmt: on
|
||||
):
|
||||
"""
|
||||
@@ -66,7 +59,7 @@ def profile(model: str, inputs: Optional[Path] = None, n_texts: int = 10000) ->
|
||||
|
||||
with msg.loading("Loading IMDB dataset via ml_datasets..."):
|
||||
imdb_train, _ = ml_datasets.imdb(train_limit=n_texts, dev_limit=0)
|
||||
texts = [text for text, _ in imdb_train]
|
||||
texts, _ = zip(*imdb_train)
|
||||
msg.info(f"Loaded IMDB dataset and using {n_texts} examples")
|
||||
with msg.loading(f"Loading pipeline '{model}'..."):
|
||||
nlp = load_model(model)
|
||||
|
||||
+5
-24
@@ -26,30 +26,11 @@ from ._util import (
|
||||
def train_cli(
|
||||
# fmt: off
|
||||
ctx: typer.Context, # This is only used to read additional arguments
|
||||
config_path: Path = Arg(
|
||||
..., help="Path to config file", exists=True, allow_dash=True
|
||||
),
|
||||
output_path: Optional[Path] = Opt(
|
||||
None,
|
||||
"--output",
|
||||
"--output-path",
|
||||
"-o",
|
||||
help="Output directory to store trained pipeline in",
|
||||
),
|
||||
code_path: Optional[Path] = Opt(
|
||||
None,
|
||||
"--code",
|
||||
"-c",
|
||||
help="Path to Python file with additional code (registered functions) to be imported",
|
||||
),
|
||||
verbose: bool = Opt(
|
||||
False,
|
||||
"--verbose",
|
||||
"-V",
|
||||
"-VV",
|
||||
help="Display more information for debugging purposes",
|
||||
),
|
||||
use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
|
||||
config_path: Path = Arg(..., help="Path to config file", exists=True, allow_dash=True),
|
||||
output_path: Optional[Path] = Opt(None, "--output", "--output-path", "-o", help="Output directory to store trained pipeline in"),
|
||||
code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
|
||||
verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
|
||||
use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU")
|
||||
# fmt: on
|
||||
):
|
||||
"""
|
||||
|
||||
+1
-4
@@ -1,5 +1,4 @@
|
||||
"""Helpers for Python and platform compatibility."""
|
||||
|
||||
import sys
|
||||
|
||||
from thinc.util import copy_array
|
||||
@@ -35,9 +34,7 @@ else:
|
||||
try: # Python 3.8+
|
||||
import importlib.metadata as importlib_metadata
|
||||
except ImportError:
|
||||
from catalogue import ( # type: ignore[no-redef]
|
||||
_importlib_metadata as importlib_metadata, # noqa: F401
|
||||
)
|
||||
from catalogue import _importlib_metadata as importlib_metadata # type: ignore[no-redef] # noqa: F401
|
||||
|
||||
from thinc.api import Optimizer # noqa: F401
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ spaCy's built in visualization suite for dependencies and named entities.
|
||||
DOCS: https://spacy.io/api/top-level#displacy
|
||||
USAGE: https://spacy.io/usage/visualizers
|
||||
"""
|
||||
|
||||
import warnings
|
||||
from typing import Any, Callable, Dict, Iterable, Optional, Union
|
||||
|
||||
@@ -67,7 +66,7 @@ def render(
|
||||
if jupyter or (jupyter is None and is_in_jupyter()):
|
||||
# return HTML rendered by IPython display()
|
||||
# See #4840 for details on span wrapper to disable mathjax
|
||||
from IPython.display import HTML, display
|
||||
from IPython.core.display import HTML, display
|
||||
|
||||
return display(HTML('<span class="tex2jax_ignore">{}</span>'.format(html)))
|
||||
return html
|
||||
|
||||
@@ -388,7 +388,7 @@ class DependencyRenderer:
|
||||
lang=self.lang,
|
||||
)
|
||||
|
||||
def render_word(self, text: str, tag: str, lemma: Optional[str], i: int) -> str:
|
||||
def render_word(self, text: str, tag: str, lemma: str, i: int) -> str:
|
||||
"""Render individual word.
|
||||
|
||||
text (str): Word text.
|
||||
|
||||
@@ -220,7 +220,6 @@ class Warnings(metaclass=ErrorsWithCodes):
|
||||
"key attribute for vectors, configure it through Vectors(attr=) or "
|
||||
"'spacy init vectors --attr'")
|
||||
W126 = ("These keys are unsupported: {unsupported}")
|
||||
W127 = ("Not all `Language.pipe` worker processes completed successfully")
|
||||
|
||||
|
||||
class Errors(metaclass=ErrorsWithCodes):
|
||||
|
||||
@@ -5,6 +5,7 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"አፕል የዩኬን ጅምር ድርጅት በ 1 ቢሊዮን ዶላር ለመግዛት አስቧል።",
|
||||
"የራስ ገዝ መኪኖች የኢንሹራንስ ኃላፊነትን ወደ አምራቾች ያዛውራሉ",
|
||||
|
||||
@@ -60,7 +60,7 @@ _ordinal_words = [
|
||||
"አስራ ስምንተኛ",
|
||||
"አስራ ዘጠነኛ",
|
||||
"ሃያኛ",
|
||||
"ሰላሳኛአርባኛ",
|
||||
"ሰላሳኛ" "አርባኛ",
|
||||
"አምሳኛ",
|
||||
"ስድሳኛ",
|
||||
"ሰባኛ",
|
||||
|
||||
@@ -4,6 +4,7 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"Bu bir cümlədir.",
|
||||
"Necəsən?",
|
||||
|
||||
@@ -3,7 +3,6 @@ References:
|
||||
https://github.com/Alir3z4/stop-words - Original list, serves as a base.
|
||||
https://postvai.com/books/stop-dumi.pdf - Additions to the original list in order to improve it.
|
||||
"""
|
||||
|
||||
STOP_WORDS = set(
|
||||
"""
|
||||
а автентичен аз ако ала
|
||||
|
||||
@@ -5,4 +5,5 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = ["তুই খুব ভালো", "আজ আমরা ডাক্তার দেখতে যাবো", "আমি জানি না "]
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
from ...language import BaseDefaults, Language
|
||||
from .lex_attrs import LEX_ATTRS
|
||||
from .stop_words import STOP_WORDS
|
||||
|
||||
|
||||
class TibetanDefaults(BaseDefaults):
|
||||
lex_attr_getters = LEX_ATTRS
|
||||
stop_words = STOP_WORDS
|
||||
|
||||
|
||||
class Tibetan(Language):
|
||||
lang = "bo"
|
||||
Defaults = TibetanDefaults
|
||||
|
||||
|
||||
__all__ = ["Tibetan"]
|
||||
@@ -1,15 +0,0 @@
|
||||
"""
|
||||
Example sentences to test spaCy and its language models.
|
||||
|
||||
>>> from spacy.lang.bo.examples import sentences
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
sentences = [
|
||||
"དོན་དུ་རྒྱ་མཚོ་བླ་མ་ཞེས་བྱ་ཞིང༌།",
|
||||
"ཏཱ་ལའི་ཞེས་པ་ནི་སོག་སྐད་ཡིན་པ་དེ་བོད་སྐད་དུ་རྒྱ་མཚོའི་དོན་དུ་འཇུག",
|
||||
"སོག་པོ་ཨལ་ཐན་རྒྱལ་པོས་རྒྱལ་དབང་བསོད་ནམས་རྒྱ་མཚོར་ཆེ་བསྟོད་ཀྱི་མཚན་གསོལ་བ་ཞིག་ཡིན་ཞིང༌།",
|
||||
"རྗེས་སུ་རྒྱལ་བ་དགེ་འདུན་གྲུབ་དང༌། དགེ་འདུན་རྒྱ་མཚོ་སོ་སོར་ཡང་ཏཱ་ལའི་བླ་མའི་སྐུ་ཕྲེང་དང་པོ་དང༌།",
|
||||
"གཉིས་པའི་མཚན་དེ་གསོལ་ཞིང༌།༸རྒྱལ་དབང་སྐུ་ཕྲེང་ལྔ་པས་དགའ་ལྡན་ཕོ་བྲང་གི་སྲིད་དབང་བཙུགས་པ་ནས་ཏཱ་ལའི་བླ་མ་ནི་བོད་ཀྱི་ཆོས་སྲིད་གཉིས་ཀྱི་དབུ་ཁྲིད་དུ་གྱུར་ཞིང་།",
|
||||
"ད་ལྟའི་བར་ཏཱ་ལའི་བླ་མ་སྐུ་ཕྲེང་བཅུ་བཞི་བྱོན་ཡོད།",
|
||||
]
|
||||
@@ -1,65 +0,0 @@
|
||||
from ...attrs import LIKE_NUM
|
||||
|
||||
# reference 1: https://en.wikipedia.org/wiki/Tibetan_numerals
|
||||
|
||||
_num_words = [
|
||||
"ཀླད་ཀོར་",
|
||||
"གཅིག་",
|
||||
"གཉིས་",
|
||||
"གསུམ་",
|
||||
"བཞི་",
|
||||
"ལྔ་",
|
||||
"དྲུག་",
|
||||
"བདུན་",
|
||||
"བརྒྱད་",
|
||||
"དགུ་",
|
||||
"བཅུ་",
|
||||
"བཅུ་གཅིག་",
|
||||
"བཅུ་གཉིས་",
|
||||
"བཅུ་གསུམ་",
|
||||
"བཅུ་བཞི་",
|
||||
"བཅུ་ལྔ་",
|
||||
"བཅུ་དྲུག་",
|
||||
"བཅུ་བདུན་",
|
||||
"བཅུ་པརྒྱད",
|
||||
"བཅུ་དགུ་",
|
||||
"ཉི་ཤུ་",
|
||||
"སུམ་ཅུ",
|
||||
"བཞི་བཅུ",
|
||||
"ལྔ་བཅུ",
|
||||
"དྲུག་ཅུ",
|
||||
"བདུན་ཅུ",
|
||||
"བརྒྱད་ཅུ",
|
||||
"དགུ་བཅུ",
|
||||
"བརྒྱ་",
|
||||
"སྟོང་",
|
||||
"ཁྲི་",
|
||||
"ས་ཡ་",
|
||||
" བྱེ་བ་",
|
||||
"དུང་ཕྱུར་",
|
||||
"ཐེར་འབུམ་",
|
||||
"ཐེར་འབུམ་ཆེན་པོ་",
|
||||
"ཁྲག་ཁྲིག་",
|
||||
"ཁྲག་ཁྲིག་ཆེན་པོ་",
|
||||
]
|
||||
|
||||
|
||||
def like_num(text):
|
||||
"""
|
||||
Check if text resembles a number
|
||||
"""
|
||||
if text.startswith(("+", "-", "±", "~")):
|
||||
text = text[1:]
|
||||
text = text.replace(",", "").replace(".", "")
|
||||
if text.isdigit():
|
||||
return True
|
||||
if text.count("/") == 1:
|
||||
num, denom = text.split("/")
|
||||
if num.isdigit() and denom.isdigit():
|
||||
return True
|
||||
if text in _num_words:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
LEX_ATTRS = {LIKE_NUM: like_num}
|
||||
@@ -1,198 +0,0 @@
|
||||
# Source: https://zenodo.org/records/10148636
|
||||
|
||||
STOP_WORDS = set(
|
||||
"""
|
||||
འི་
|
||||
།
|
||||
དུ་
|
||||
གིས་
|
||||
སོགས་
|
||||
ཏེ
|
||||
གི་
|
||||
རྣམས་
|
||||
ནི
|
||||
ཀུན་
|
||||
ཡི་
|
||||
འདི
|
||||
ཀྱི་
|
||||
སྙེད་
|
||||
པས་
|
||||
གཞན་
|
||||
ཀྱིས་
|
||||
ཡི
|
||||
ལ
|
||||
ནི་
|
||||
དང་
|
||||
སོགས
|
||||
ཅིང་
|
||||
ར
|
||||
དུ
|
||||
མི་
|
||||
སུ་
|
||||
བཅས་
|
||||
ཡོངས་
|
||||
ལས
|
||||
ཙམ་
|
||||
གྱིས་
|
||||
དེ་
|
||||
ཡང་
|
||||
མཐའ་དག་
|
||||
ཏུ་
|
||||
ཉིད་
|
||||
ས
|
||||
ཏེ་
|
||||
གྱི་
|
||||
སྤྱི
|
||||
དེ
|
||||
ཀ་
|
||||
ཡིན་
|
||||
ཞིང་
|
||||
འདི་
|
||||
རུང་
|
||||
རང་
|
||||
ཞིག་
|
||||
སྟེ
|
||||
སྟེ་
|
||||
ན་རེ
|
||||
ངམ
|
||||
ཤིང་
|
||||
དག་
|
||||
ཏོ
|
||||
རེ་
|
||||
འང་
|
||||
ཀྱང་
|
||||
ལགས་པ
|
||||
ཚུ
|
||||
དོ
|
||||
ཡིན་པ
|
||||
རེ
|
||||
ན་རེ་
|
||||
ཨེ་
|
||||
ཚང་མ
|
||||
ཐམས་ཅད་
|
||||
དམ་
|
||||
འོ་
|
||||
ཅིག་
|
||||
གྱིན་
|
||||
ཡིན
|
||||
ན
|
||||
ཁོ་ན་
|
||||
འམ་
|
||||
ཀྱིན་
|
||||
ལོ
|
||||
ཀྱིས
|
||||
བས་
|
||||
ལགས་
|
||||
ཤིག
|
||||
གིས
|
||||
ཀི་
|
||||
སྣ་ཚོགས་
|
||||
རྣམས
|
||||
སྙེད་པ
|
||||
ཡིས་
|
||||
གྱི
|
||||
གི
|
||||
བམ་
|
||||
ཤིག་
|
||||
རེ་རེ་
|
||||
ནམ
|
||||
མིན་
|
||||
ནམ་
|
||||
ངམ་
|
||||
རུ་
|
||||
འགའ་
|
||||
ཀུན
|
||||
ཤས་
|
||||
ཏུ
|
||||
ཡིས
|
||||
གིན་
|
||||
གམ་
|
||||
འོ
|
||||
ཡིན་པ་
|
||||
མིན
|
||||
ལགས
|
||||
གྱིས
|
||||
ཅང་
|
||||
འགའ
|
||||
སམ་
|
||||
ཞིག
|
||||
འང
|
||||
ལས་ཆེ་
|
||||
འཕྲལ་
|
||||
བར་
|
||||
རུ
|
||||
དང
|
||||
ཡ
|
||||
འག
|
||||
སམ
|
||||
ཀ
|
||||
ཅུང་ཟད་
|
||||
ཅིག
|
||||
ཉིད
|
||||
དུ་མ
|
||||
མ
|
||||
ཡིན་བ
|
||||
འམ
|
||||
མམ
|
||||
དམ
|
||||
དག
|
||||
ཁོ་ན
|
||||
ཀྱི
|
||||
ལམ
|
||||
ཕྱི་
|
||||
ནང་
|
||||
ཙམ
|
||||
ནོ་
|
||||
སོ་
|
||||
རམ་
|
||||
བོ་
|
||||
ཨང་
|
||||
ཕྱི
|
||||
ཏོ་
|
||||
ཚོ
|
||||
ལ་ལ་
|
||||
ཚོ་
|
||||
ཅིང
|
||||
མ་གི་
|
||||
གེ
|
||||
གོ
|
||||
ཡིན་ལུགས་
|
||||
རོ་
|
||||
བོ
|
||||
ལགས་པ་
|
||||
པས
|
||||
རབ་
|
||||
འི
|
||||
རམ
|
||||
བས
|
||||
གཞན
|
||||
སྙེད་པ་
|
||||
འབའ་
|
||||
མཾ་
|
||||
པོ
|
||||
ག་
|
||||
ག
|
||||
གམ
|
||||
སྤྱི་
|
||||
བམ
|
||||
མོ་
|
||||
ཙམ་པ་
|
||||
ཤ་སྟག་
|
||||
མམ་
|
||||
རེ་རེ
|
||||
སྙེད
|
||||
ཏམ་
|
||||
ངོ
|
||||
གྲང་
|
||||
ཏ་རེ
|
||||
ཏམ
|
||||
ཁ་
|
||||
ངེ་
|
||||
ཅོག་
|
||||
རིལ་
|
||||
ཉུང་ཤས་
|
||||
གིང་
|
||||
ཚ་
|
||||
ཀྱང
|
||||
""".split()
|
||||
)
|
||||
@@ -5,6 +5,7 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"Apple està buscant comprar una startup del Regne Unit per mil milions de dòlars",
|
||||
"Els cotxes autònoms deleguen la responsabilitat de l'assegurança als seus fabricants",
|
||||
|
||||
@@ -277,10 +277,10 @@ _currency = (
|
||||
# These expressions contain various unicode variations, including characters
|
||||
# used in Chinese (see #1333, #1340, #1351) – unless there are cross-language
|
||||
# conflicts, spaCy's base tokenizer should handle all of those by default
|
||||
_punct = r"… …… , : ; \! \? ¿ ؟ ¡ \( \) \[ \] \{ \} < > _ # \* & 。 ？ ！ ， 、 ； ： ～ · । ، ۔ ؛ ٪"
|
||||
_quotes = (
|
||||
r'\' " ” “ ` ‘ ´ ’ ‚ , „ » « 「 」 『 』 （ ） 〔 〕 【 】 《 》 〈 〉 〈 〉 ⟦ ⟧'
|
||||
_punct = (
|
||||
r"… …… , : ; \! \? ¿ ؟ ¡ \( \) \[ \] \{ \} < > _ # \* & 。 ？ ！ ， 、 ； ： ～ · । ، ۔ ؛ ٪"
|
||||
)
|
||||
_quotes = r'\' " ” “ ` ‘ ´ ’ ‚ , „ » « 「 」 『 』 （ ） 〔 〕 【 】 《 》 〈 〉 〈 〉 ⟦ ⟧'
|
||||
_hyphens = "- – — -- --- —— ~"
|
||||
|
||||
# Various symbols like dingbats, but also emoji
|
||||
|
||||
@@ -4,6 +4,7 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"Máma mele maso.",
|
||||
"Příliš žluťoučký kůň úpěl ďábelské ódy.",
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
Tokenizer Exceptions.
|
||||
Source: https://forkortelse.dk/ and various others.
|
||||
"""
|
||||
|
||||
from ...symbols import NORM, ORTH
|
||||
from ...util import update_exc
|
||||
from ..tokenizer_exceptions import BASE_EXCEPTIONS
|
||||
|
||||
@@ -5,6 +5,7 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"Die ganze Stadt ist ein Startup: Shenzhen ist das Silicon Valley für Hardware-Firmen",
|
||||
"Wie deutsche Startups die Technologie vorantreiben wollen: Künstliche Intelligenz",
|
||||
|
||||
@@ -5,6 +5,7 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"Z tym stwori so wuměnjenje a zakład za dalše wobdźěłanje přez analyzu tekstoweje struktury a semantisku anotaciju a z tym tež za tu předstajenu digitalnu online-wersiju.",
|
||||
"Mi so tu jara derje spodoba.",
|
||||
|
||||
@@ -128,6 +128,7 @@ _other_exc = {
|
||||
_exc.update(_other_exc)
|
||||
|
||||
for h in range(1, 12 + 1):
|
||||
|
||||
for period in ["π.μ.", "πμ"]:
|
||||
_exc[f"{h}{period}"] = [
|
||||
{ORTH: f"{h}"},
|
||||
|
||||
@@ -5,6 +5,7 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"Apple is looking at buying U.K. startup for $1 billion",
|
||||
"Autonomous cars shift insurance liability toward manufacturers",
|
||||
|
||||
@@ -5,6 +5,7 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"Apple está buscando comprar una startup del Reino Unido por mil millones de dólares.",
|
||||
"Los coches autónomos delegan la responsabilidad del seguro en sus fabricantes.",
|
||||
|
||||
@@ -415,10 +415,7 @@ class SpanishLemmatizer(Lemmatizer):
|
||||
else:
|
||||
rule = self.select_rule("verb", features)
|
||||
verb_lemma = self.lemmatize_verb(
|
||||
verb,
|
||||
features - {"PronType=Prs"}, # type: ignore[operator]
|
||||
rule,
|
||||
index, # type: ignore[operator]
|
||||
verb, features - {"PronType=Prs"}, rule, index # type: ignore[operator]
|
||||
)[0]
|
||||
pron_lemmas = []
|
||||
for pron in prons:
|
||||
|
||||
@@ -5,6 +5,7 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"این یک جمله نمونه می باشد.",
|
||||
"قرار ما، امروز ساعت ۲:۳۰ بعدازظهر هست!",
|
||||
|
||||
@@ -611,8 +611,8 @@ narrative_ends = ["هام", "های", "ه", "هایم", "هاید",
|
||||
present_ends = ["م", "ی", "د", "یم", "ید", "ند"]
|
||||
|
||||
# special case of '#هست':
|
||||
VERBS_EXC.update(dict.fromkeys(["هست" + end for end in simple_ends], "هست"))
|
||||
VERBS_EXC.update(dict.fromkeys(["نیست" + end for end in simple_ends], "هست"))
|
||||
VERBS_EXC.update({conj: "هست" for conj in ["هست" + end for end in simple_ends]})
|
||||
VERBS_EXC.update({conj: "هست" for conj in ["نیست" + end for end in simple_ends]})
|
||||
|
||||
for verb_root in verb_roots:
|
||||
conjugations = []
|
||||
@@ -648,4 +648,4 @@ for verb_root in verb_roots:
|
||||
)
|
||||
)
|
||||
|
||||
VERBS_EXC.update(dict.fromkeys(conjugations, (past,) if past else present))
|
||||
VERBS_EXC.update({conj: (past,) if past else present for conj in conjugations})
|
||||
|
||||
@@ -100,9 +100,9 @@ conj_contraction_negations = [
|
||||
("eivat", "eivät"),
|
||||
("eivät", "eivät"),
|
||||
]
|
||||
for base_lower, base_norm in conj_contraction_bases:
|
||||
for (base_lower, base_norm) in conj_contraction_bases:
|
||||
for base in [base_lower, base_lower.title()]:
|
||||
for suffix, suffix_norm in conj_contraction_negations:
|
||||
for (suffix, suffix_norm) in conj_contraction_negations:
|
||||
_exc[base + suffix] = [
|
||||
{ORTH: base, NORM: base_norm},
|
||||
{ORTH: suffix, NORM: suffix_norm},
|
||||
|
||||
@@ -5,7 +5,9 @@ from ..tokenizer_exceptions import BASE_EXCEPTIONS
|
||||
_exc = {}
|
||||
|
||||
for orth in [
|
||||
"Chr.",
|
||||
"apr.",
|
||||
"atm.",
|
||||
"aug.",
|
||||
"avgr.",
|
||||
"árg.",
|
||||
@@ -15,21 +17,53 @@ for orth in [
|
||||
"blaðkv.",
|
||||
"blm.",
|
||||
"blaðm.",
|
||||
"blaðstj.",
|
||||
"blkv.",
|
||||
"blm.",
|
||||
"bls.",
|
||||
"blstj.",
|
||||
"blaðstj.",
|
||||
"cand.",
|
||||
"dagf.",
|
||||
"des.",
|
||||
"dkr.",
|
||||
"dr.",
|
||||
"e.Kr.",
|
||||
"eint.",
|
||||
"ex.",
|
||||
"exam.",
|
||||
"f.",
|
||||
"f.Kr.",
|
||||
"fa.",
|
||||
"fam.",
|
||||
"feb.",
|
||||
"febr.",
|
||||
"ff.",
|
||||
"fl.",
|
||||
"form.",
|
||||
"frí.",
|
||||
"fyrrv.",
|
||||
"góðk.",
|
||||
"h.m.",
|
||||
"hósd.",
|
||||
"innt.",
|
||||
"jan.",
|
||||
"kap.",
|
||||
"kgl.",
|
||||
"kl.",
|
||||
"kr.",
|
||||
"leyg.",
|
||||
"m.a.",
|
||||
"mðr.",
|
||||
"m.o.",
|
||||
"m.ø.",
|
||||
"mia.",
|
||||
"mik.",
|
||||
"min.",
|
||||
"mió.",
|
||||
"mán.",
|
||||
"mðr.",
|
||||
"nov.",
|
||||
"nr.",
|
||||
"nto.",
|
||||
"nov.",
|
||||
@@ -43,12 +77,25 @@ for orth in [
|
||||
"o.o.",
|
||||
"o.s.fr.",
|
||||
"o.tíl.",
|
||||
"o.u.",
|
||||
"o.ø.",
|
||||
"okt.",
|
||||
"omf.",
|
||||
"ph.d.",
|
||||
"phil.",
|
||||
"pr.",
|
||||
"pst.",
|
||||
"ritstj.",
|
||||
"s.",
|
||||
"sb.",
|
||||
"sbr.",
|
||||
"sbrt.",
|
||||
"sek.",
|
||||
"sep.",
|
||||
"sept.",
|
||||
"serst.",
|
||||
"smb.",
|
||||
"smbr.",
|
||||
"sms.",
|
||||
"smst.",
|
||||
"smb.",
|
||||
@@ -58,14 +105,25 @@ for orth in [
|
||||
"sept.",
|
||||
"spf.",
|
||||
"spsk.",
|
||||
"stk.",
|
||||
"sunnud.",
|
||||
"t.",
|
||||
"t.d.",
|
||||
"t.e.",
|
||||
"t.o.v.",
|
||||
"t.s.",
|
||||
"t.s.s.",
|
||||
"tlf.",
|
||||
"t.v.s",
|
||||
"t.v.s.",
|
||||
"tel.",
|
||||
"tils.",
|
||||
"tlf.",
|
||||
"tsk.",
|
||||
"t.o.v.",
|
||||
"t.d.",
|
||||
"tús.",
|
||||
"týs.",
|
||||
"uml.",
|
||||
"ums.",
|
||||
"uppl.",
|
||||
@@ -75,13 +133,22 @@ for orth in [
|
||||
"útl.",
|
||||
"útr.",
|
||||
"vanl.",
|
||||
"upprfr.",
|
||||
"v.",
|
||||
"v.h.",
|
||||
"v.m.",
|
||||
"v.ø.o.",
|
||||
"vanl.",
|
||||
"viðm.",
|
||||
"viðv.",
|
||||
"vm.",
|
||||
"v.m.",
|
||||
"á.Kr.",
|
||||
"árg.",
|
||||
"ávís.",
|
||||
"útg.",
|
||||
"útl.",
|
||||
"útr.",
|
||||
]:
|
||||
_exc[orth] = [{ORTH: orth}]
|
||||
capitalized = orth.capitalize()
|
||||
|
||||
@@ -5,6 +5,7 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"Apple cherche à acheter une start-up anglaise pour 1 milliard de dollars",
|
||||
"Les voitures autonomes déplacent la responsabilité de l'assurance vers les constructeurs",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import List, Tuple
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
from ...pipeline import Lemmatizer
|
||||
from ...tokens import Token
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
from typing import Optional
|
||||
|
||||
from ...language import BaseDefaults, Language
|
||||
from .stop_words import STOP_WORDS
|
||||
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
|
||||
|
||||
|
||||
class ScottishDefaults(BaseDefaults):
|
||||
tokenizer_exceptions = TOKENIZER_EXCEPTIONS
|
||||
stop_words = STOP_WORDS
|
||||
|
||||
|
||||
class Scottish(Language):
|
||||
lang = "gd"
|
||||
Defaults = ScottishDefaults
|
||||
|
||||
|
||||
__all__ = ["Scottish"]
|
||||
@@ -1,386 +0,0 @@
|
||||
STOP_WORDS = set(
|
||||
"""
|
||||
'ad
|
||||
'ar
|
||||
'd # iad
|
||||
'g # ag
|
||||
'ga
|
||||
'gam
|
||||
'gan
|
||||
'gar
|
||||
'gur
|
||||
'm # am
|
||||
'n # an
|
||||
'n seo
|
||||
'na
|
||||
'nad
|
||||
'nam
|
||||
'nan
|
||||
'nar
|
||||
'nuair
|
||||
'nur
|
||||
's
|
||||
'sa
|
||||
'san
|
||||
'sann
|
||||
'se
|
||||
'sna
|
||||
a
|
||||
a'
|
||||
a'd # agad
|
||||
a'm # agam
|
||||
a-chèile
|
||||
a-seo
|
||||
a-sin
|
||||
a-siud
|
||||
a chionn
|
||||
a chionn 's
|
||||
a chèile
|
||||
a chéile
|
||||
a dh'
|
||||
a h-uile
|
||||
a seo
|
||||
ac' # aca
|
||||
aca
|
||||
aca-san
|
||||
acasan
|
||||
ach
|
||||
ag
|
||||
agad
|
||||
agad-sa
|
||||
agads'
|
||||
agadsa
|
||||
agaibh
|
||||
agaibhse
|
||||
againn
|
||||
againne
|
||||
agam
|
||||
agam-sa
|
||||
agams'
|
||||
agamsa
|
||||
agus
|
||||
aice
|
||||
aice-se
|
||||
aicese
|
||||
aig
|
||||
aig' # aige
|
||||
aige
|
||||
aige-san
|
||||
aigesan
|
||||
air
|
||||
air-san
|
||||
air neo
|
||||
airsan
|
||||
am
|
||||
an
|
||||
an seo
|
||||
an sin
|
||||
an siud
|
||||
an uair
|
||||
ann
|
||||
ann a
|
||||
ann a'
|
||||
ann a shin
|
||||
ann am
|
||||
ann an
|
||||
annad
|
||||
annam
|
||||
annam-s'
|
||||
annamsa
|
||||
anns
|
||||
anns an
|
||||
annta
|
||||
aon
|
||||
ar
|
||||
as
|
||||
asad
|
||||
asda
|
||||
asta
|
||||
b'
|
||||
bho
|
||||
bhon
|
||||
bhuaidhe # bhuaithe
|
||||
bhuainn
|
||||
bhuaipe
|
||||
bhuaithe
|
||||
bhuapa
|
||||
bhur
|
||||
brì
|
||||
bu
|
||||
c'à
|
||||
car son
|
||||
carson
|
||||
cha
|
||||
chan
|
||||
chionn
|
||||
choir
|
||||
chon
|
||||
chun
|
||||
chèile
|
||||
chéile
|
||||
chòir
|
||||
cia mheud
|
||||
ciamar
|
||||
co-dhiubh
|
||||
cuide
|
||||
cuin
|
||||
cuin'
|
||||
cuine
|
||||
cà
|
||||
cà'
|
||||
càil
|
||||
càit
|
||||
càit'
|
||||
càite
|
||||
cò
|
||||
cò mheud
|
||||
có
|
||||
d'
|
||||
da
|
||||
de
|
||||
dh'
|
||||
dha
|
||||
dhaibh
|
||||
dhaibh-san
|
||||
dhaibhsan
|
||||
dhan
|
||||
dhasan
|
||||
dhe
|
||||
dhen
|
||||
dheth
|
||||
dhi
|
||||
dhiom
|
||||
dhiot
|
||||
dhith
|
||||
dhiubh
|
||||
dhomh
|
||||
dhomh-s'
|
||||
dhomhsa
|
||||
dhu'sa # dhut-sa
|
||||
dhuibh
|
||||
dhuibhse
|
||||
dhuinn
|
||||
dhuinne
|
||||
dhuit
|
||||
dhut
|
||||
dhutsa
|
||||
dhut-sa
|
||||
dhà
|
||||
dhà-san
|
||||
dhàsan
|
||||
dhòmhsa
|
||||
diubh
|
||||
do
|
||||
docha
|
||||
don
|
||||
dà
|
||||
dè
|
||||
dè mar
|
||||
dé
|
||||
dé mar
|
||||
dòch'
|
||||
dòcha
|
||||
e
|
||||
eadar
|
||||
eatarra
|
||||
eatorra
|
||||
eile
|
||||
esan
|
||||
fa
|
||||
far
|
||||
feud
|
||||
fhad
|
||||
fheudar
|
||||
fhearr
|
||||
fhein
|
||||
fheudar
|
||||
fheàrr
|
||||
fhèin
|
||||
fhéin
|
||||
fhìn
|
||||
fo
|
||||
fodha
|
||||
fodhainn
|
||||
foipe
|
||||
fon
|
||||
fèin
|
||||
ga
|
||||
gach
|
||||
gam
|
||||
gan
|
||||
ge brith
|
||||
ged
|
||||
gu
|
||||
gu dè
|
||||
gu ruige
|
||||
gun
|
||||
gur
|
||||
gus
|
||||
i
|
||||
iad
|
||||
iadsan
|
||||
innte
|
||||
is
|
||||
ise
|
||||
le
|
||||
leam
|
||||
leam-sa
|
||||
leamsa
|
||||
leat
|
||||
leat-sa
|
||||
leatha
|
||||
leatsa
|
||||
leibh
|
||||
leis
|
||||
leis-san
|
||||
leoth'
|
||||
leotha
|
||||
leotha-san
|
||||
linn
|
||||
m'
|
||||
m'a
|
||||
ma
|
||||
mac
|
||||
man
|
||||
mar
|
||||
mas
|
||||
mathaid
|
||||
mi
|
||||
mis'
|
||||
mise
|
||||
mo
|
||||
mu
|
||||
mu 'n
|
||||
mun
|
||||
mur
|
||||
mura
|
||||
mus
|
||||
na
|
||||
na b'
|
||||
na bu
|
||||
na iad
|
||||
nach
|
||||
nad
|
||||
nam
|
||||
nan
|
||||
nar
|
||||
nas
|
||||
neo
|
||||
no
|
||||
nuair
|
||||
o
|
||||
o'n
|
||||
oir
|
||||
oirbh
|
||||
oirbh-se
|
||||
oirnn
|
||||
oirnne
|
||||
oirre
|
||||
on
|
||||
orm
|
||||
orm-sa
|
||||
ormsa
|
||||
orra
|
||||
orra-san
|
||||
orrasan
|
||||
ort
|
||||
os
|
||||
r'
|
||||
ri
|
||||
ribh
|
||||
rinn
|
||||
ris
|
||||
rithe
|
||||
rithe-se
|
||||
rium
|
||||
rium-sa
|
||||
riums'
|
||||
riumsa
|
||||
riut
|
||||
riuth'
|
||||
riutha
|
||||
riuthasan
|
||||
ro
|
||||
ro'n
|
||||
roimh
|
||||
roimhe
|
||||
romhainn
|
||||
romham
|
||||
romhpa
|
||||
ron
|
||||
ruibh
|
||||
ruinn
|
||||
ruinne
|
||||
sa
|
||||
san
|
||||
sann
|
||||
se
|
||||
seach
|
||||
seo
|
||||
seothach
|
||||
shin
|
||||
sibh
|
||||
sibh-se
|
||||
sibhse
|
||||
sin
|
||||
sineach
|
||||
sinn
|
||||
sinne
|
||||
siod
|
||||
siodach
|
||||
siud
|
||||
siudach
|
||||
sna # ann an
|
||||
sè
|
||||
t'
|
||||
tarsaing
|
||||
tarsainn
|
||||
tarsuinn
|
||||
thar
|
||||
thoigh
|
||||
thro
|
||||
thu
|
||||
thuc'
|
||||
thuca
|
||||
thugad
|
||||
thugaibh
|
||||
thugainn
|
||||
thugam
|
||||
thugamsa
|
||||
thuice
|
||||
thuige
|
||||
thus'
|
||||
thusa
|
||||
timcheall
|
||||
toigh
|
||||
toil
|
||||
tro
|
||||
tro' # troimh
|
||||
troimh
|
||||
troimhe
|
||||
tron
|
||||
tu
|
||||
tusa
|
||||
uair
|
||||
ud
|
||||
ugaibh
|
||||
ugam-s'
|
||||
ugam-sa
|
||||
uice
|
||||
uige
|
||||
uige-san
|
||||
umad
|
||||
unnta # ann an
|
||||
ur
|
||||
urrainn
|
||||
à
|
||||
às
|
||||
àsan
|
||||
á
|
||||
ás
|
||||
è
|
||||
ì
|
||||
ò
|
||||
ó
|
||||
""".split("\n")
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,6 +5,7 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"ἐρᾷ μὲν ἁγνὸς οὐρανὸς τρῶσαι χθόνα, ἔρως δὲ γαῖαν λαμβάνει γάμου τυχεῖν·",
|
||||
"εὐδαίμων Χαρίτων καὶ Μελάνιππος ἔφυ, θείας ἁγητῆρες ἐφαμερίοις φιλότατος.",
|
||||
|
||||
@@ -5,6 +5,7 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"લોકશાહી એ સરકારનું એક એવું તંત્ર છે જ્યાં નાગરિકો મત દ્વારા સત્તાનો ઉપયોગ કરે છે.",
|
||||
"તે ગુજરાત રાજ્યના ધરમપુર શહેરમાં આવેલું હતું",
|
||||
|
||||
@@ -5,6 +5,7 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"סין מקימה קרן של 440 מיליון דולר להשקעה בהייטק בישראל",
|
||||
'רה"מ הודיע כי יחרים טקס בחסותו',
|
||||
|
||||
@@ -5,6 +5,7 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"एप्पल 1 अरब डॉलर के लिए यू.के. स्टार्टअप खरीदने पर विचार कर रहा है।",
|
||||
"स्वायत्त कारें निर्माताओं की ओर बीमा दायित्व रखतीं हैं।",
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
The list of Croatian lemmas was extracted from the reldi-tagger repository (https://github.com/clarinsi/reldi-tagger).
|
||||
Reldi-tagger is licensed under the Apache 2.0 licence.
|
||||
Reldi-tagger is licesned under the Apache 2.0 licence.
|
||||
|
||||
@InProceedings{ljubesic16-new,
|
||||
author = {Nikola Ljubešić and Filip Klubička and Željko Agić and Ivo-Pavao Jazbec},
|
||||
@@ -12,4 +12,4 @@ Reldi-tagger is licensed under the Apache 2.0 licence.
|
||||
publisher = {European Language Resources Association (ELRA)},
|
||||
address = {Paris, France},
|
||||
isbn = {978-2-9517408-9-1}
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"To běšo wjelgin raźone a jo se wót luźi derje pśiwzeło. Tak som dožywiła wjelgin",
|
||||
"Jogo pśewóźowarce stej groniłej, až how w serbskich stronach njama Santa Claus nic pytaś.",
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
from typing import Callable, Optional
|
||||
|
||||
from thinc.api import Model
|
||||
|
||||
from ...language import BaseDefaults, Language
|
||||
from .lemmatizer import HaitianCreoleLemmatizer
|
||||
from .lex_attrs import LEX_ATTRS
|
||||
from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES
|
||||
from .stop_words import STOP_WORDS
|
||||
from .syntax_iterators import SYNTAX_ITERATORS
|
||||
from .tag_map import TAG_MAP
|
||||
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
|
||||
|
||||
|
||||
class HaitianCreoleDefaults(BaseDefaults):
|
||||
tokenizer_exceptions = TOKENIZER_EXCEPTIONS
|
||||
prefixes = TOKENIZER_PREFIXES
|
||||
infixes = TOKENIZER_INFIXES
|
||||
suffixes = TOKENIZER_SUFFIXES
|
||||
lex_attr_getters = LEX_ATTRS
|
||||
syntax_iterators = SYNTAX_ITERATORS
|
||||
stop_words = STOP_WORDS
|
||||
tag_map = TAG_MAP
|
||||
|
||||
|
||||
class HaitianCreole(Language):
|
||||
lang = "ht"
|
||||
Defaults = HaitianCreoleDefaults
|
||||
|
||||
|
||||
@HaitianCreole.factory(
|
||||
"lemmatizer",
|
||||
assigns=["token.lemma"],
|
||||
default_config={
|
||||
"model": None,
|
||||
"mode": "rule",
|
||||
"overwrite": False,
|
||||
"scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"},
|
||||
},
|
||||
default_score_weights={"lemma_acc": 1.0},
|
||||
)
|
||||
def make_lemmatizer(
|
||||
nlp: Language,
|
||||
model: Optional[Model],
|
||||
name: str,
|
||||
mode: str,
|
||||
overwrite: bool,
|
||||
scorer: Optional[Callable],
|
||||
):
|
||||
return HaitianCreoleLemmatizer(
|
||||
nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["HaitianCreole"]
|
||||
@@ -1,17 +0,0 @@
|
||||
"""
|
||||
Example sentences to test spaCy and its language models.
|
||||
|
||||
>>> from spacy.lang.ht.examples import sentences
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
sentences = [
|
||||
"Apple ap panse achte yon demaraj nan Wayòm Ini pou $1 milya dola",
|
||||
"Machin otonòm fè responsablite asirans lan ale sou men fabrikan yo",
|
||||
"San Francisco ap konsidere entèdi robo ki livre sou twotwa yo",
|
||||
"Lond se yon gwo vil nan Wayòm Ini",
|
||||
"Kote ou ye?",
|
||||
"Kilès ki prezidan Lafrans?",
|
||||
"Ki kapital Etazini?",
|
||||
"Kile Barack Obama te fèt?",
|
||||
]
|
||||
@@ -1,50 +0,0 @@
|
||||
from typing import List, Tuple
|
||||
|
||||
from ...pipeline import Lemmatizer
|
||||
from ...tokens import Token
|
||||
|
||||
|
||||
class HaitianCreoleLemmatizer(Lemmatizer):
|
||||
"""
|
||||
Minimal Haitian Creole lemmatizer.
|
||||
Returns a word's base form based on rules and lookup,
|
||||
or defaults to the original form.
|
||||
"""
|
||||
|
||||
def is_base_form(self, token: Token) -> bool:
|
||||
morph = token.morph.to_dict()
|
||||
upos = token.pos_.lower()
|
||||
|
||||
# Consider unmarked forms to be base
|
||||
if upos in {"noun", "verb", "adj", "adv"}:
|
||||
if not morph:
|
||||
return True
|
||||
if upos == "noun" and morph.get("Number") == "Sing":
|
||||
return True
|
||||
if upos == "verb" and morph.get("VerbForm") == "Inf":
|
||||
return True
|
||||
if upos == "adj" and morph.get("Degree") == "Pos":
|
||||
return True
|
||||
return False
|
||||
|
||||
def rule_lemmatize(self, token: Token) -> List[str]:
|
||||
string = token.text.lower()
|
||||
pos = token.pos_.lower()
|
||||
cache_key = (token.orth, token.pos)
|
||||
if cache_key in self.cache:
|
||||
return self.cache[cache_key]
|
||||
|
||||
forms = []
|
||||
|
||||
# fallback rule: just return lowercased form
|
||||
forms.append(string)
|
||||
|
||||
self.cache[cache_key] = forms
|
||||
return forms
|
||||
|
||||
@classmethod
|
||||
def get_lookups_config(cls, mode: str) -> Tuple[List[str], List[str]]:
|
||||
if mode == "rule":
|
||||
required = ["lemma_lookup", "lemma_rules", "lemma_exc", "lemma_index"]
|
||||
return (required, [])
|
||||
return super().get_lookups_config(mode)
|
||||
@@ -1,81 +0,0 @@
|
||||
from ...attrs import LIKE_NUM, NORM
|
||||
|
||||
# Cardinal numbers in Creole
|
||||
_num_words = set(
|
||||
"""
|
||||
zewo youn en de twa kat senk sis sèt uit nèf dis
|
||||
onz douz trèz katoz kenz sèz disèt dizwit diznèf
|
||||
vent trant karant sinkant swasant swasann-dis
|
||||
san mil milyon milya
|
||||
""".split()
|
||||
)
|
||||
|
||||
# Ordinal numbers in Creole (some are French-influenced, some simplified)
|
||||
_ordinal_words = set(
|
||||
"""
|
||||
premye dezyèm twazyèm katryèm senkyèm sizyèm sètvyèm uitvyèm nèvyèm dizyèm
|
||||
onzèm douzyèm trèzyèm katozyèm kenzèm sèzyèm disetyèm dizwityèm diznèvyèm
|
||||
ventyèm trantyèm karantyèm sinkantyèm swasantyèm
|
||||
swasann-disyèm santyèm milyèm milyonnyèm milyadyèm
|
||||
""".split()
|
||||
)
|
||||
|
||||
NORM_MAP = {
|
||||
"'m": "mwen",
|
||||
"'w": "ou",
|
||||
"'l": "li",
|
||||
"'n": "nou",
|
||||
"'y": "yo",
|
||||
"’m": "mwen",
|
||||
"’w": "ou",
|
||||
"’l": "li",
|
||||
"’n": "nou",
|
||||
"’y": "yo",
|
||||
"m": "mwen",
|
||||
"n": "nou",
|
||||
"l": "li",
|
||||
"y": "yo",
|
||||
"w": "ou",
|
||||
"t": "te",
|
||||
"k": "ki",
|
||||
"p": "pa",
|
||||
"M": "Mwen",
|
||||
"N": "Nou",
|
||||
"L": "Li",
|
||||
"Y": "Yo",
|
||||
"W": "Ou",
|
||||
"T": "Te",
|
||||
"K": "Ki",
|
||||
"P": "Pa",
|
||||
}
|
||||
|
||||
|
||||
def like_num(text):
|
||||
text = text.strip().lower()
|
||||
if text.startswith(("+", "-", "±", "~")):
|
||||
text = text[1:]
|
||||
text = text.replace(",", "").replace(".", "")
|
||||
if text.isdigit():
|
||||
return True
|
||||
if text.count("/") == 1:
|
||||
num, denom = text.split("/")
|
||||
if num.isdigit() and denom.isdigit():
|
||||
return True
|
||||
if text in _num_words:
|
||||
return True
|
||||
if text in _ordinal_words:
|
||||
return True
|
||||
# Handle things like "3yèm", "10yèm", "25yèm", etc.
|
||||
if text.endswith("yèm") and text[:-3].isdigit():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def norm_custom(text):
|
||||
return NORM_MAP.get(text, text.lower())
|
||||
|
||||
|
||||
LEX_ATTRS = {
|
||||
LIKE_NUM: like_num,
|
||||
NORM: norm_custom,
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
from ..char_classes import (
|
||||
ALPHA,
|
||||
ALPHA_LOWER,
|
||||
ALPHA_UPPER,
|
||||
CONCAT_QUOTES,
|
||||
HYPHENS,
|
||||
LIST_ELLIPSES,
|
||||
LIST_ICONS,
|
||||
LIST_PUNCT,
|
||||
LIST_QUOTES,
|
||||
merge_chars,
|
||||
)
|
||||
|
||||
ELISION = "'’".replace(" ", "")
|
||||
|
||||
_prefixes_elision = "m n l y t k w"
|
||||
_prefixes_elision += " " + _prefixes_elision.upper()
|
||||
|
||||
TOKENIZER_PREFIXES = (
|
||||
LIST_PUNCT
|
||||
+ LIST_QUOTES
|
||||
+ [
|
||||
r"(?:({pe})[{el}])(?=[{a}])".format(
|
||||
a=ALPHA, el=ELISION, pe=merge_chars(_prefixes_elision)
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
TOKENIZER_SUFFIXES = (
|
||||
LIST_PUNCT
|
||||
+ LIST_QUOTES
|
||||
+ LIST_ELLIPSES
|
||||
+ [
|
||||
r"(?<=[0-9])%", # numbers like 10%
|
||||
r"(?<=[0-9])(?:{h})".format(h=HYPHENS), # hyphens after numbers
|
||||
r"(?<=[{a}])['’]".format(a=ALPHA), # apostrophes after letters
|
||||
r"(?<=[{a}])['’][mwlnytk](?=\s|$)".format(a=ALPHA), # contractions
|
||||
r"(?<=[{a}0-9])\)", # right parenthesis after letter/number
|
||||
r"(?<=[{a}])\.(?=\s|$)".format(
|
||||
a=ALPHA
|
||||
), # period after letter if space or end of string
|
||||
r"(?<=\))[\.\?!]", # punctuation immediately after right parenthesis
|
||||
]
|
||||
)
|
||||
|
||||
TOKENIZER_INFIXES = (
|
||||
LIST_ELLIPSES
|
||||
+ LIST_ICONS
|
||||
+ [
|
||||
r"(?<=[0-9])[+\-\*^](?=[0-9-])",
|
||||
r"(?<=[{al}{q}])\.(?=[{au}{q}])".format(
|
||||
al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES
|
||||
),
|
||||
r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA),
|
||||
r"(?<=[{a}0-9])(?:{h})(?=[{a}])".format(a=ALPHA, h=HYPHENS),
|
||||
r"(?<=[{a}][{el}])(?=[{a}])".format(a=ALPHA, el=ELISION),
|
||||
]
|
||||
)
|
||||
@@ -1,49 +0,0 @@
|
||||
STOP_WORDS = set(
|
||||
"""
|
||||
a ak an ankò ant apre ap atò avan avanlè
|
||||
byen bò byenke
|
||||
|
||||
chak
|
||||
|
||||
de depi deja deja
|
||||
|
||||
e en epi èske
|
||||
|
||||
fò fòk
|
||||
|
||||
gen genyen
|
||||
|
||||
ki kisa kilès kote koukou konsa konbyen konn konnen kounye kouman
|
||||
|
||||
la l laa le lè li lye lò
|
||||
|
||||
m m' mwen
|
||||
|
||||
nan nap nou n'
|
||||
|
||||
ou oumenm
|
||||
|
||||
pa paske pami pandan pito pou pral preske pwiske
|
||||
|
||||
se selman si sou sòt
|
||||
|
||||
ta tap tankou te toujou tou tan tout toutotan twòp tèl
|
||||
|
||||
w w' wi wè
|
||||
|
||||
y y' yo yon yonn
|
||||
|
||||
non o oh eh
|
||||
|
||||
sa san si swa si
|
||||
|
||||
men mèsi oswa osinon
|
||||
|
||||
""".split()
|
||||
)
|
||||
|
||||
# Add common contractions, with and without apostrophe variants
|
||||
contractions = ["m'", "n'", "w'", "y'", "l'", "t'", "k'"]
|
||||
for apostrophe in ["'", "’", "‘"]:
|
||||
for word in contractions:
|
||||
STOP_WORDS.add(word.replace("'", apostrophe))
|
||||
@@ -1,74 +0,0 @@
|
||||
from typing import Iterator, Tuple, Union
|
||||
|
||||
from ...errors import Errors
|
||||
from ...symbols import NOUN, PRON, PROPN
|
||||
from ...tokens import Doc, Span
|
||||
|
||||
|
||||
def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]:
|
||||
"""
|
||||
Detect base noun phrases from a dependency parse for Haitian Creole.
|
||||
Works on both Doc and Span objects.
|
||||
"""
|
||||
|
||||
# Core nominal dependencies common in Haitian Creole
|
||||
labels = [
|
||||
"nsubj",
|
||||
"obj",
|
||||
"obl",
|
||||
"nmod",
|
||||
"appos",
|
||||
"ROOT",
|
||||
]
|
||||
|
||||
# Modifiers to optionally include in chunk (to the right)
|
||||
post_modifiers = ["compound", "flat", "flat:name", "fixed"]
|
||||
|
||||
doc = doclike.doc
|
||||
if not doc.has_annotation("DEP"):
|
||||
raise ValueError(Errors.E029)
|
||||
|
||||
np_deps = {doc.vocab.strings.add(label) for label in labels}
|
||||
np_mods = {doc.vocab.strings.add(mod) for mod in post_modifiers}
|
||||
conj_label = doc.vocab.strings.add("conj")
|
||||
np_label = doc.vocab.strings.add("NP")
|
||||
adp_pos = doc.vocab.strings.add("ADP")
|
||||
cc_pos = doc.vocab.strings.add("CCONJ")
|
||||
|
||||
prev_end = -1
|
||||
for i, word in enumerate(doclike):
|
||||
if word.pos not in (NOUN, PROPN, PRON):
|
||||
continue
|
||||
if word.left_edge.i <= prev_end:
|
||||
continue
|
||||
|
||||
if word.dep in np_deps:
|
||||
right_end = word
|
||||
# expand to include known modifiers to the right
|
||||
for child in word.rights:
|
||||
if child.dep in np_mods:
|
||||
right_end = child.right_edge
|
||||
elif child.pos == NOUN:
|
||||
right_end = child.right_edge
|
||||
|
||||
left_index = word.left_edge.i
|
||||
# Skip prepositions at the start
|
||||
if word.left_edge.pos == adp_pos:
|
||||
left_index += 1
|
||||
|
||||
prev_end = right_end.i
|
||||
yield left_index, right_end.i + 1, np_label
|
||||
|
||||
elif word.dep == conj_label:
|
||||
head = word.head
|
||||
while head.dep == conj_label and head.head.i < head.i:
|
||||
head = head.head
|
||||
if head.dep in np_deps:
|
||||
left_index = word.left_edge.i
|
||||
if word.left_edge.pos == cc_pos:
|
||||
left_index += 1
|
||||
prev_end = word.i
|
||||
yield left_index, word.i + 1, np_label
|
||||
|
||||
|
||||
SYNTAX_ITERATORS = {"noun_chunks": noun_chunks}
|
||||
@@ -1,39 +0,0 @@
|
||||
from spacy.symbols import (
|
||||
ADJ,
|
||||
ADP,
|
||||
ADV,
|
||||
AUX,
|
||||
CCONJ,
|
||||
DET,
|
||||
INTJ,
|
||||
NOUN,
|
||||
NUM,
|
||||
PART,
|
||||
PRON,
|
||||
PROPN,
|
||||
PUNCT,
|
||||
SCONJ,
|
||||
SYM,
|
||||
VERB,
|
||||
X,
|
||||
)
|
||||
|
||||
TAG_MAP = {
|
||||
"NOUN": {"pos": NOUN},
|
||||
"VERB": {"pos": VERB},
|
||||
"AUX": {"pos": AUX},
|
||||
"ADJ": {"pos": ADJ},
|
||||
"ADV": {"pos": ADV},
|
||||
"PRON": {"pos": PRON},
|
||||
"DET": {"pos": DET},
|
||||
"ADP": {"pos": ADP},
|
||||
"SCONJ": {"pos": SCONJ},
|
||||
"CCONJ": {"pos": CCONJ},
|
||||
"PART": {"pos": PART},
|
||||
"INTJ": {"pos": INTJ},
|
||||
"NUM": {"pos": NUM},
|
||||
"PROPN": {"pos": PROPN},
|
||||
"PUNCT": {"pos": PUNCT},
|
||||
"SYM": {"pos": SYM},
|
||||
"X": {"pos": X},
|
||||
}
|
||||
@@ -1,126 +0,0 @@
|
||||
from spacy.symbols import NORM, ORTH
|
||||
|
||||
|
||||
def make_variants(base, first_norm, second_orth, second_norm):
|
||||
return {
|
||||
base: [
|
||||
{ORTH: base.split("'")[0] + "'", NORM: first_norm},
|
||||
{ORTH: second_orth, NORM: second_norm},
|
||||
],
|
||||
base.capitalize(): [
|
||||
{
|
||||
ORTH: base.split("'")[0].capitalize() + "'",
|
||||
NORM: first_norm.capitalize(),
|
||||
},
|
||||
{ORTH: second_orth, NORM: second_norm},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
TOKENIZER_EXCEPTIONS = {"Dr.": [{ORTH: "Dr."}]}
|
||||
|
||||
# Apostrophe forms
|
||||
TOKENIZER_EXCEPTIONS.update(make_variants("m'ap", "mwen", "ap", "ap"))
|
||||
TOKENIZER_EXCEPTIONS.update(make_variants("n'ap", "nou", "ap", "ap"))
|
||||
TOKENIZER_EXCEPTIONS.update(make_variants("l'ap", "li", "ap", "ap"))
|
||||
TOKENIZER_EXCEPTIONS.update(make_variants("y'ap", "yo", "ap", "ap"))
|
||||
TOKENIZER_EXCEPTIONS.update(make_variants("m'te", "mwen", "te", "te"))
|
||||
TOKENIZER_EXCEPTIONS.update(make_variants("m'pral", "mwen", "pral", "pral"))
|
||||
TOKENIZER_EXCEPTIONS.update(make_variants("w'ap", "ou", "ap", "ap"))
|
||||
TOKENIZER_EXCEPTIONS.update(make_variants("k'ap", "ki", "ap", "ap"))
|
||||
TOKENIZER_EXCEPTIONS.update(make_variants("p'ap", "pa", "ap", "ap"))
|
||||
TOKENIZER_EXCEPTIONS.update(make_variants("t'ap", "te", "ap", "ap"))
|
||||
|
||||
# Non-apostrophe contractions (with capitalized variants)
|
||||
TOKENIZER_EXCEPTIONS.update(
|
||||
{
|
||||
"map": [
|
||||
{ORTH: "m", NORM: "mwen"},
|
||||
{ORTH: "ap", NORM: "ap"},
|
||||
],
|
||||
"Map": [
|
||||
{ORTH: "M", NORM: "Mwen"},
|
||||
{ORTH: "ap", NORM: "ap"},
|
||||
],
|
||||
"lem": [
|
||||
{ORTH: "le", NORM: "le"},
|
||||
{ORTH: "m", NORM: "mwen"},
|
||||
],
|
||||
"Lem": [
|
||||
{ORTH: "Le", NORM: "Le"},
|
||||
{ORTH: "m", NORM: "mwen"},
|
||||
],
|
||||
"lew": [
|
||||
{ORTH: "le", NORM: "le"},
|
||||
{ORTH: "w", NORM: "ou"},
|
||||
],
|
||||
"Lew": [
|
||||
{ORTH: "Le", NORM: "Le"},
|
||||
{ORTH: "w", NORM: "ou"},
|
||||
],
|
||||
"nap": [
|
||||
{ORTH: "n", NORM: "nou"},
|
||||
{ORTH: "ap", NORM: "ap"},
|
||||
],
|
||||
"Nap": [
|
||||
{ORTH: "N", NORM: "Nou"},
|
||||
{ORTH: "ap", NORM: "ap"},
|
||||
],
|
||||
"lap": [
|
||||
{ORTH: "l", NORM: "li"},
|
||||
{ORTH: "ap", NORM: "ap"},
|
||||
],
|
||||
"Lap": [
|
||||
{ORTH: "L", NORM: "Li"},
|
||||
{ORTH: "ap", NORM: "ap"},
|
||||
],
|
||||
"yap": [
|
||||
{ORTH: "y", NORM: "yo"},
|
||||
{ORTH: "ap", NORM: "ap"},
|
||||
],
|
||||
"Yap": [
|
||||
{ORTH: "Y", NORM: "Yo"},
|
||||
{ORTH: "ap", NORM: "ap"},
|
||||
],
|
||||
"mte": [
|
||||
{ORTH: "m", NORM: "mwen"},
|
||||
{ORTH: "te", NORM: "te"},
|
||||
],
|
||||
"Mte": [
|
||||
{ORTH: "M", NORM: "Mwen"},
|
||||
{ORTH: "te", NORM: "te"},
|
||||
],
|
||||
"mpral": [
|
||||
{ORTH: "m", NORM: "mwen"},
|
||||
{ORTH: "pral", NORM: "pral"},
|
||||
],
|
||||
"Mpral": [
|
||||
{ORTH: "M", NORM: "Mwen"},
|
||||
{ORTH: "pral", NORM: "pral"},
|
||||
],
|
||||
"wap": [
|
||||
{ORTH: "w", NORM: "ou"},
|
||||
{ORTH: "ap", NORM: "ap"},
|
||||
],
|
||||
"Wap": [
|
||||
{ORTH: "W", NORM: "Ou"},
|
||||
{ORTH: "ap", NORM: "ap"},
|
||||
],
|
||||
"kap": [
|
||||
{ORTH: "k", NORM: "ki"},
|
||||
{ORTH: "ap", NORM: "ap"},
|
||||
],
|
||||
"Kap": [
|
||||
{ORTH: "K", NORM: "Ki"},
|
||||
{ORTH: "ap", NORM: "ap"},
|
||||
],
|
||||
"tap": [
|
||||
{ORTH: "t", NORM: "te"},
|
||||
{ORTH: "ap", NORM: "ap"},
|
||||
],
|
||||
"Tap": [
|
||||
{ORTH: "T", NORM: "Te"},
|
||||
{ORTH: "ap", NORM: "ap"},
|
||||
],
|
||||
}
|
||||
)
|
||||
@@ -5,6 +5,7 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"Az Apple egy brit startup vásárlását tervezi 1 milliárd dollár értékben.",
|
||||
"San Francisco vezetése mérlegeli a járdát használó szállító robotok betiltását.",
|
||||
|
||||
@@ -11,7 +11,7 @@ from ..char_classes import (
|
||||
)
|
||||
|
||||
# removing ° from the special icons to keep e.g. 99° as one token
|
||||
_concat_icons = CONCAT_ICONS.replace("\u00b0", "")
|
||||
_concat_icons = CONCAT_ICONS.replace("\u00B0", "")
|
||||
|
||||
_currency = r"\$¢£€¥฿"
|
||||
_quotes = CONCAT_QUOTES.replace("'", "")
|
||||
|
||||
@@ -4,6 +4,7 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"Լոնդոնը Միացյալ Թագավորության մեծ քաղաք է։",
|
||||
"Ո՞վ է Ֆրանսիայի նախագահը։",
|
||||
|
||||
@@ -5,6 +5,7 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"Indonesia merupakan negara kepulauan yang kaya akan budaya.",
|
||||
"Berapa banyak warga yang dibutuhkan saat kerja bakti?",
|
||||
|
||||
@@ -156,7 +156,7 @@ for orth in [
|
||||
"S.T.",
|
||||
"S.T.Han",
|
||||
"S.Th.",
|
||||
"S.Th.IS.TI.",
|
||||
"S.Th.I" "S.TI.",
|
||||
"S.T.P.",
|
||||
"S.TrK",
|
||||
"S.Tekp.",
|
||||
@@ -210,7 +210,7 @@ for orth in [
|
||||
"hlm.",
|
||||
"i/o",
|
||||
"n.b.",
|
||||
"p.p.pjs.",
|
||||
"p.p." "pjs.",
|
||||
"s.d.",
|
||||
"tel.",
|
||||
"u.p.",
|
||||
|
||||
@@ -5,6 +5,7 @@ Example sentences to test spaCy and its language models.
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"Apple vuole comprare una startup del Regno Unito per un miliardo di dollari",
|
||||
"Le automobili a guida autonoma spostano la responsabilità assicurativa verso i produttori",
|
||||
|
||||
@@ -32,6 +32,7 @@ split_mode = null
|
||||
"""
|
||||
|
||||
|
||||
@registry.tokenizers("spacy.ja.JapaneseTokenizer")
|
||||
def create_tokenizer(split_mode: Optional[str] = None):
|
||||
def japanese_tokenizer_factory(nlp):
|
||||
return JapaneseTokenizer(nlp.vocab, split_mode=split_mode)
|
||||
@@ -61,7 +62,7 @@ class JapaneseTokenizer(DummyTokenizer):
|
||||
zip(*dtokens) if dtokens else [[]] * 7
|
||||
)
|
||||
sub_tokens_list = list(sub_tokens_list)
|
||||
doc = Doc(self.vocab, words=list(words), spaces=spaces)
|
||||
doc = Doc(self.vocab, words=words, spaces=spaces)
|
||||
next_pos = None # for bi-gram rules
|
||||
for idx, (token, dtoken) in enumerate(zip(doc, dtokens)):
|
||||
token.tag_ = dtoken.tag
|
||||
@@ -102,9 +103,9 @@ class JapaneseTokenizer(DummyTokenizer):
|
||||
token.dictionary_form(), # lemma
|
||||
token.normalized_form(),
|
||||
token.reading_form(),
|
||||
(
|
||||
sub_tokens_list[idx] if sub_tokens_list else None
|
||||
), # user_data['sub_tokens']
|
||||
sub_tokens_list[idx]
|
||||
if sub_tokens_list
|
||||
else None, # user_data['sub_tokens']
|
||||
)
|
||||
for idx, token in enumerate(sudachipy_tokens)
|
||||
if len(token.surface()) > 0
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user