Soham Kukreti
660f49c879
LLMConfig.__init__ checks PROVIDER_MODELS_PREFIXES when api_token=None. If the provider prefix isn't found, it silently falls through to the else branch and overwrites self.provider with DEFAULT_PROVIDER (openai/gpt-4o), meaning any bedrock/* model string was being replaced before the LLM call was even made. This broke supported Bedrock auth methods when api_token is not passed in the LLMConfig Only passing api_token=<bearer_token> explicitly worked, because the truthy api_token bypassed the prefix check entirely. Adding "bedrock": None to PROVIDER_MODELS_PREFIXES keeps self.provider intact so the correct Bedrock provider is used. The actual auth (SigV4 signing or Bearer header) is handled downstream based on what credentials are available in the environment.
156 lines
5.1 KiB
Python
156 lines
5.1 KiB
Python
import os
|
|
from dotenv import load_dotenv
|
|
|
|
load_dotenv() # Load environment variables from .env file
|
|
|
|
# Default provider, ONLY used when the extraction strategy is LLMExtractionStrategy
|
|
DEFAULT_PROVIDER = "openai/gpt-4o"
|
|
DEFAULT_PROVIDER_API_KEY = "OPENAI_API_KEY"
|
|
MODEL_REPO_BRANCH = "new-release-0.0.2"
|
|
# Provider-model dictionary, ONLY used when the extraction strategy is LLMExtractionStrategy
|
|
PROVIDER_MODELS = {
|
|
"ollama/llama3": "no-token-needed", # Any model from Ollama no need for API token
|
|
"groq/llama3-70b-8192": os.getenv("GROQ_API_KEY"),
|
|
"groq/llama3-8b-8192": os.getenv("GROQ_API_KEY"),
|
|
"openai/gpt-4o-mini": os.getenv("OPENAI_API_KEY"),
|
|
"openai/gpt-4o": os.getenv("OPENAI_API_KEY"),
|
|
"openai/o1-mini": os.getenv("OPENAI_API_KEY"),
|
|
"openai/o1-preview": os.getenv("OPENAI_API_KEY"),
|
|
"openai/o3-mini": os.getenv("OPENAI_API_KEY"),
|
|
"openai/o3-mini-high": os.getenv("OPENAI_API_KEY"),
|
|
"anthropic/claude-3-haiku-20240307": os.getenv("ANTHROPIC_API_KEY"),
|
|
"anthropic/claude-3-opus-20240229": os.getenv("ANTHROPIC_API_KEY"),
|
|
"anthropic/claude-3-sonnet-20240229": os.getenv("ANTHROPIC_API_KEY"),
|
|
"anthropic/claude-3-5-sonnet-20240620": os.getenv("ANTHROPIC_API_KEY"),
|
|
"gemini/gemini-pro": os.getenv("GEMINI_API_KEY"),
|
|
'gemini/gemini-1.5-pro': os.getenv("GEMINI_API_KEY"),
|
|
'gemini/gemini-2.0-flash': os.getenv("GEMINI_API_KEY"),
|
|
'gemini/gemini-2.0-flash-exp': os.getenv("GEMINI_API_KEY"),
|
|
'gemini/gemini-2.0-flash-lite-preview-02-05': os.getenv("GEMINI_API_KEY"),
|
|
"deepseek/deepseek-chat": os.getenv("DEEPSEEK_API_KEY"),
|
|
}
|
|
PROVIDER_MODELS_PREFIXES = {
|
|
"ollama": "no-token-needed", # Any model from Ollama no need for API token
|
|
"groq": os.getenv("GROQ_API_KEY"),
|
|
"openai": os.getenv("OPENAI_API_KEY"),
|
|
"anthropic": os.getenv("ANTHROPIC_API_KEY"),
|
|
"gemini": os.getenv("GEMINI_API_KEY"),
|
|
"deepseek": os.getenv("DEEPSEEK_API_KEY"),
|
|
"bedrock": None, # Bedrock uses AWS credential chain (SigV4) or explicit api_token for bearer auth
|
|
}
|
|
|
|
# Chunk token threshold
|
|
CHUNK_TOKEN_THRESHOLD = 2**11 # 2048 tokens
|
|
OVERLAP_RATE = 0.1
|
|
WORD_TOKEN_RATE = 1.3
|
|
|
|
# Threshold for the minimum number of word in a HTML tag to be considered
|
|
MIN_WORD_THRESHOLD = 1
|
|
IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD = 1
|
|
|
|
IMPORTANT_ATTRS = ["src", "href", "alt", "title", "width", "height", "class", "id", "rowspan", "colspan"]
|
|
ONLY_TEXT_ELIGIBLE_TAGS = [
|
|
"b",
|
|
"i",
|
|
"u",
|
|
"span",
|
|
"del",
|
|
"ins",
|
|
"sub",
|
|
"sup",
|
|
"strong",
|
|
"em",
|
|
"code",
|
|
"kbd",
|
|
"var",
|
|
"s",
|
|
"q",
|
|
"abbr",
|
|
"cite",
|
|
"dfn",
|
|
"time",
|
|
"small",
|
|
"mark",
|
|
]
|
|
SOCIAL_MEDIA_DOMAINS = [
|
|
"facebook.com",
|
|
"twitter.com",
|
|
"x.com",
|
|
"linkedin.com",
|
|
"instagram.com",
|
|
"pinterest.com",
|
|
"tiktok.com",
|
|
"snapchat.com",
|
|
"reddit.com",
|
|
]
|
|
|
|
# Threshold for the Image extraction - Range is 1 to 6
|
|
# Images are scored based on point based system, to filter based on usefulness. Points are assigned
|
|
# to each image based on the following aspects.
|
|
# If either height or width exceeds 150px
|
|
# If image size is greater than 10Kb
|
|
# If alt property is set
|
|
# If image format is in jpg, png or webp
|
|
# If image is in the first half of the total images extracted from the page
|
|
IMAGE_SCORE_THRESHOLD = 2
|
|
|
|
MAX_METRICS_HISTORY = 1000
|
|
|
|
NEED_MIGRATION = True
|
|
URL_LOG_SHORTEN_LENGTH = 30
|
|
SHOW_DEPRECATION_WARNINGS = True
|
|
SCREENSHOT_HEIGHT_TRESHOLD = 10000
|
|
PAGE_TIMEOUT = 60000
|
|
DOWNLOAD_PAGE_TIMEOUT = 60000
|
|
|
|
# Delimiter for concatenating multiple HTML examples in schema generation
|
|
HTML_EXAMPLE_DELIMITER = "=== HTML EXAMPLE {index} ==="
|
|
|
|
# Global user settings with descriptions and default values
|
|
USER_SETTINGS = {
|
|
"DEFAULT_LLM_PROVIDER": {
|
|
"default": "openai/gpt-4o",
|
|
"description": "Default LLM provider in 'company/model' format (e.g., 'openai/gpt-4o', 'anthropic/claude-3-sonnet')",
|
|
"type": "string"
|
|
},
|
|
"DEFAULT_LLM_PROVIDER_TOKEN": {
|
|
"default": "",
|
|
"description": "API token for the default LLM provider",
|
|
"type": "string",
|
|
"secret": True
|
|
},
|
|
"VERBOSE": {
|
|
"default": False,
|
|
"description": "Enable verbose output for all commands",
|
|
"type": "boolean"
|
|
},
|
|
"BROWSER_HEADLESS": {
|
|
"default": True,
|
|
"description": "Run browser in headless mode by default",
|
|
"type": "boolean"
|
|
},
|
|
"BROWSER_TYPE": {
|
|
"default": "chromium",
|
|
"description": "Default browser type (chromium or firefox)",
|
|
"type": "string",
|
|
"options": ["chromium", "firefox"]
|
|
},
|
|
"CACHE_MODE": {
|
|
"default": "bypass",
|
|
"description": "Default cache mode (bypass, use, or refresh)",
|
|
"type": "string",
|
|
"options": ["bypass", "use", "refresh"]
|
|
},
|
|
"USER_AGENT_MODE": {
|
|
"default": "default",
|
|
"description": "Default user agent mode (default, random, or mobile)",
|
|
"type": "string",
|
|
"options": ["default", "random", "mobile"]
|
|
},
|
|
"JSON_ENSURE_ASCII": {
|
|
"default": True,
|
|
"description": "Whether to escape non-ASCII characters in JSON output (False preserves Unicode like 'š', True escapes as '\\u0161')",
|
|
"type": "boolean"
|
|
}
|
|
}
|