mindsdb--mindsdb/setup.py

import os
import glob

from setuptools import find_packages, setup


class Deps:
    pkgs = []
    pkgs_exclude = ["tests", "tests.*"]
    new_links = []
    extras = {}


about = {}
with open("mindsdb/__about__.py") as fp:
    exec(fp.read(), about)


with open("README.md", "r", encoding="utf8") as fh:
    long_description = fh.read()


def filter_requirements(lines: list) -> list:
    """
    Filter out comments, empty lines, and editable installs from requirement lines.
    We are have generated requirements files with transitive dependencies.
    """
    result = []
    for line in lines:
        line = line.strip()
        if not line or line.startswith("#") or line.startswith("-e "):
            continue
        result.append(line)
    return result


def expand_requirements_links(requirements: list) -> list:
    """Expand requirements that contain links to other requirement files"""
    to_add = []
    to_remove = []

    for requirement in requirements:
        if requirement.startswith("-r "):
            if os.path.exists(requirement.split()[1]):
                with open(requirement.split()[1]) as fh:
                    to_add += expand_requirements_links(filter_requirements(fh.read().splitlines()))
            to_remove.append(requirement)

    for req in to_remove:
        requirements.remove(req)
    for req in to_add:
        requirements.append(req)

    return list(set(requirements))  # Remove duplicates


def define_deps():
    """Reads requirements.txt requirements-extra.txt files and preprocess it
    to be feed into setuptools.

    This is the only possible way (we found)
    how requirements.txt can be reused in setup.py
    using dependencies from private github repositories.

    Links must be appendend by `-{StringWithAtLeastOneNumber}`
    or something like that, so e.g. `-9231` works as well as
    `1.1.0`. This is ignored by the setuptools, but has to be there.

    Warnings:
        to make pip respect the links, you have to use
        `--process-dependency-links` switch. So e.g.:
        `pip install --process-dependency-links {git-url}`

    Returns:
         list of packages, extras and dependency links.
    """
    with open(os.path.normpath("requirements/requirements.txt")) as req_file:
        defaults = filter_requirements(req_file.read().splitlines())

    links = []
    requirements = []
    for r in defaults:
        if "git+https" in r:
            pkg = r.split("#")[-1]
            links.append(r + "-9876543210")
            requirements.append(pkg.replace("egg=", ""))
        else:
            requirements.append(r.strip())

    extra_requirements = {}
    full_requirements = []
    for fn in os.listdir(os.path.normpath("./requirements")):
        if fn.startswith("requirements-") and fn.endswith(".txt"):
            extra_name = fn.replace("requirements-", "").replace(".txt", "")
            with open(os.path.normpath(f"./requirements/{fn}")) as fp:
                extra = filter_requirements(fp.read().splitlines())
            extra_requirements[extra_name] = extra
            full_requirements += extra

    extra_requirements["all_extras"] = list(set(full_requirements))
    extra_requirements["all"] = list(set(full_requirements))

    full_handlers_requirements = []
    handlers_dir_path = os.path.normpath("./mindsdb/integrations/handlers")
    for fn in os.listdir(handlers_dir_path):
        if os.path.isdir(os.path.join(handlers_dir_path, fn)) and fn.endswith("_handler"):
            base_extra_name = fn.replace("_handler", "")
            extra_requirements[base_extra_name] = []
            for req_file_path in glob.glob(os.path.join(handlers_dir_path, fn, "requirements*.txt")):
                extra_name = base_extra_name
                file_name = os.path.basename(req_file_path)
                if file_name != "requirements.txt":
                    extra_name += "-" + file_name.replace("requirements_", "").replace(".txt", "")

                # If requirements.txt in our handler folder, import them as our extra's requirements
                with open(req_file_path) as fp:
                    extra = expand_requirements_links(filter_requirements(fp.read().splitlines()))

                extra_requirements[extra_name] = extra
                full_handlers_requirements += extra

    extra_requirements["all_handlers_extras"] = list(set(full_handlers_requirements))

    with open(os.path.normpath("requirements/requirements-opentelemetry.txt")) as req_file:
        extra_requirements["opentelemetry"] = filter_requirements(req_file.read().splitlines())

    with open(os.path.normpath("requirements/requirements-langfuse.txt")) as req_file:
        extra_requirements["langfuse"] = filter_requirements(req_file.read().splitlines())

    Deps.pkgs = requirements
    Deps.extras = extra_requirements
    Deps.new_links = links

    return Deps


deps = define_deps()

setup(
    name=about["__title__"],
    version=about["__version__"],
    url=about["__github__"],
    download_url=about["__pypi__"],
    license=about["__license__"],
    author=about["__author__"],
    author_email=about["__email__"],
    description=about["__description__"],
    long_description=long_description,
    long_description_content_type="text/markdown",
    packages=find_packages(exclude=deps.pkgs_exclude),
    install_requires=deps.pkgs,
    dependency_links=deps.new_links,
    extras_require=deps.extras,
    include_package_data=True,
    classifiers=[
        "Programming Language :: Python :: 3",
        "Operating System :: OS Independent",
    ],
    python_requires=">=3.10,<3.14",
)