Input documents

In [1]:

# Copyright (c) 2024 Microsoft Corporation.
# Licensed under the MIT License.

# Copyright (c) 2024 Microsoft Corporation. # Licensed under the MIT License.

Example of indexing from an existing in-memory dataframe¶

Newer versions of GraphRAG let you submit a dataframe directly instead of running through the input processing step. This notebook demonstrates with regular or update runs.

If performing an update, the assumption is that your dataframe contains only the new documents to add to the index.

In [2]:

from pathlib import Path
from pprint import pprint

import graphrag.api as api
import pandas as pd
from graphrag.config.load_config import load_config
from graphrag.index.typing.pipeline_run_result import PipelineRunResult

from pathlib import Path from pprint import pprint import graphrag.api as api import pandas as pd from graphrag.config.load_config import load_config from graphrag.index.typing.pipeline_run_result import PipelineRunResult

In [3]:

PROJECT_DIRECTORY = "<your project directory>"
UPDATE = False
FILENAME = "new_documents.parquet" if UPDATE else "<original_documents>.parquet"
inputs = pd.read_parquet(f"{PROJECT_DIRECTORY}/input/{FILENAME}")
# Only the bare minimum for input. These are the same fields that would be present after the load_input_documents workflow
inputs = inputs.loc[:, ["id", "title", "text", "creation_date"]]

PROJECT_DIRECTORY = "<your project directory>" UPDATE = False FILENAME = "new_documents.parquet" if UPDATE else "<original_documents>.parquet" inputs = pd.read_parquet(f"{PROJECT_DIRECTORY}/input/{FILENAME}") # Only the bare minimum for input. These are the same fields that would be present after the load_input_documents workflow inputs = inputs.loc[:, ["id", "title", "text", "creation_date"]]

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
Cell In[3], line 4
      1 PROJECT_DIRECTORY = "<your project directory>"
      2 UPDATE = False
      3 FILENAME = "new_documents.parquet" if UPDATE else "<original_documents>.parquet"
----> 4 inputs = pd.read_parquet(f"{PROJECT_DIRECTORY}/input/{FILENAME}")
      5 # Only the bare minimum for input. These are the same fields that would be present after the load_input_documents workflow
      6 inputs = inputs.loc[:, ["id", "title", "text", "creation_date"]]

File ~/work/graphrag/graphrag/.venv/lib/python3.13/site-packages/pandas/io/parquet.py:671, in read_parquet(path, engine, columns, storage_options, dtype_backend, filesystem, filters, to_pandas_kwargs, **kwargs)
    668 impl = get_engine(engine)
    669 check_dtype_backend(dtype_backend)
--> 671 return impl.read(
    672     path,
    673     columns=columns,
    674     filters=filters,
    675     storage_options=storage_options,
    676     dtype_backend=dtype_backend,
    677     filesystem=filesystem,
    678     to_pandas_kwargs=to_pandas_kwargs,
    679     **kwargs,
    680 )

File ~/work/graphrag/graphrag/.venv/lib/python3.13/site-packages/pandas/io/parquet.py:253, in PyArrowImpl.read(self, path, columns, filters, dtype_backend, storage_options, filesystem, to_pandas_kwargs, **kwargs)
    240 def read(
    241     self,
    242     path,
   (...)    249     **kwargs,
    250 ) -> DataFrame:
    251     kwargs["use_pandas_metadata"] = True
--> 253     path_or_handle, handles, filesystem = _get_path_or_handle(
    254         path,
    255         filesystem,
    256         storage_options=storage_options,
    257         mode="rb",
    258     )
    259     try:
    260         pa_table = self.api.parquet.read_table(
    261             path_or_handle,
    262             columns=columns,
   (...)    265             **kwargs,
    266         )

File ~/work/graphrag/graphrag/.venv/lib/python3.13/site-packages/pandas/io/parquet.py:141, in _get_path_or_handle(path, fs, storage_options, mode, is_dir)
    131 handles = None
    132 if (
    133     not fs
    134     and not is_dir
   (...)    139     # fsspec resources can also point to directories
    140     # this branch is used for example when reading from non-fsspec URLs
--> 141     handles = get_handle(
    142         path_or_handle, mode, is_text=False, storage_options=storage_options
    143     )
    144     fs = None
    145     path_or_handle = handles.handle

File ~/work/graphrag/graphrag/.venv/lib/python3.13/site-packages/pandas/io/common.py:939, in get_handle(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)
    930         handle = open(
    931             handle,
    932             ioargs.mode,
   (...)    935             newline="",
    936         )
    937     else:
    938         # Binary mode
--> 939         handle = open(handle, ioargs.mode)
    940     handles.append(handle)
    942 # Convert BytesIO or file objects passed with an encoding

FileNotFoundError: [Errno 2] No such file or directory: '<your project directory>/input/<original_documents>.parquet'

Generate a GraphRagConfig object¶

In [4]:

graphrag_config = load_config(Path(PROJECT_DIRECTORY))

graphrag_config = load_config(Path(PROJECT_DIRECTORY))

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
Cell In[4], line 1
----> 1 graphrag_config = load_config(Path(PROJECT_DIRECTORY))

File ~/work/graphrag/graphrag/packages/graphrag/graphrag/config/load_config.py:43, in load_config(root_dir, cli_overrides)
     14 def load_config(
     15     root_dir: str | Path,
     16     cli_overrides: dict[str, Any] | None = None,
     17 ) -> GraphRagConfig:
     18     """Load configuration from a file.
     19 
     20     Parameters
   (...)     41         If there are pydantic validation errors when instantiating the config.
     42     """
---> 43     return lc(
     44         config_initializer=GraphRagConfig,
     45         config_path=root_dir,
     46         overrides=cli_overrides,
     47     )

File ~/work/graphrag/graphrag/packages/graphrag-common/graphrag_common/config/load_config.py:172, in load_config(config_initializer, config_path, overrides, set_cwd, parse_env_vars, load_dot_env_file, dot_env_path, config_parser, file_encoding)
    117 """Load configuration from a file.
    118 
    119 Parameters
   (...)    169     - If the parser fails to parse the configuration text.
    170 """
    171 config_path = Path(config_path).resolve() if config_path else Path.cwd()
--> 172 config_path = _get_config_file_path(config_path)
    174 file_contents = config_path.read_text(encoding=file_encoding)
    176 if parse_env_vars:

File ~/work/graphrag/graphrag/packages/graphrag-common/graphrag_common/config/load_config.py:38, in _get_config_file_path(config_dir_or_file)
     36 if not config_dir_or_file.is_dir():
     37     msg = f"Invalid config path: {config_dir_or_file} is not a directory"
---> 38     raise FileNotFoundError(msg)
     40 for file in _default_config_files:
     41     if (config_dir_or_file / file).is_file():

FileNotFoundError: Invalid config path: /home/runner/work/graphrag/graphrag/docs/examples_notebooks/<your project directory> is not a directory

Indexing API¶

Indexing is the process of ingesting raw text data and constructing a knowledge graph. GraphRAG currently supports plaintext (.txt) and .csv file formats.

Build an index¶

In [5]:

index_result: list[PipelineRunResult] = await api.build_index(
    config=graphrag_config, input_documents=inputs, is_update_run=UPDATE
)

# index_result is a list of workflows that make up the indexing pipeline that was run
for workflow_result in index_result:
    status = f"error\n{workflow_result.errors}" if workflow_result.errors else "success"
    print(f"Workflow Name: {workflow_result.workflow}\tStatus: {status}")

index_result: list[PipelineRunResult] = await api.build_index( config=graphrag_config, input_documents=inputs, is_update_run=UPDATE ) # index_result is a list of workflows that make up the indexing pipeline that was run for workflow_result in index_result: status = f"error\n{workflow_result.errors}" if workflow_result.errors else "success" print(f"Workflow Name: {workflow_result.workflow}\tStatus: {status}")

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Cell In[5], line 2
      1 index_result: list[PipelineRunResult] = await api.build_index(
----> 2     config=graphrag_config, input_documents=inputs, is_update_run=UPDATE
      3 )
      4 
      5 # index_result is a list of workflows that make up the indexing pipeline that was run

NameError: name 'graphrag_config' is not defined

Query an index¶

To query an index, several index files must first be read into memory and passed to the query API.

In [6]:

entities = pd.read_parquet(f"{PROJECT_DIRECTORY}/output/entities.parquet")
communities = pd.read_parquet(f"{PROJECT_DIRECTORY}/output/communities.parquet")
community_reports = pd.read_parquet(
    f"{PROJECT_DIRECTORY}/output/community_reports.parquet"
)

response, context = await api.global_search(
    config=graphrag_config,
    entities=entities,
    communities=communities,
    community_reports=community_reports,
    community_level=2,
    dynamic_community_selection=False,
    response_type="Multiple Paragraphs",
    query="What are the top five themes of the dataset?",
)

entities = pd.read_parquet(f"{PROJECT_DIRECTORY}/output/entities.parquet") communities = pd.read_parquet(f"{PROJECT_DIRECTORY}/output/communities.parquet") community_reports = pd.read_parquet( f"{PROJECT_DIRECTORY}/output/community_reports.parquet" ) response, context = await api.global_search( config=graphrag_config, entities=entities, communities=communities, community_reports=community_reports, community_level=2, dynamic_community_selection=False, response_type="Multiple Paragraphs", query="What are the top five themes of the dataset?", )

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
Cell In[6], line 1
----> 1 entities = pd.read_parquet(f"{PROJECT_DIRECTORY}/output/entities.parquet")
      2 communities = pd.read_parquet(f"{PROJECT_DIRECTORY}/output/communities.parquet")
      3 community_reports = pd.read_parquet(
      4     f"{PROJECT_DIRECTORY}/output/community_reports.parquet"

File ~/work/graphrag/graphrag/.venv/lib/python3.13/site-packages/pandas/io/parquet.py:671, in read_parquet(path, engine, columns, storage_options, dtype_backend, filesystem, filters, to_pandas_kwargs, **kwargs)
    668 impl = get_engine(engine)
    669 check_dtype_backend(dtype_backend)
--> 671 return impl.read(
    672     path,
    673     columns=columns,
    674     filters=filters,
    675     storage_options=storage_options,
    676     dtype_backend=dtype_backend,
    677     filesystem=filesystem,
    678     to_pandas_kwargs=to_pandas_kwargs,
    679     **kwargs,
    680 )

File ~/work/graphrag/graphrag/.venv/lib/python3.13/site-packages/pandas/io/parquet.py:253, in PyArrowImpl.read(self, path, columns, filters, dtype_backend, storage_options, filesystem, to_pandas_kwargs, **kwargs)
    240 def read(
    241     self,
    242     path,
   (...)    249     **kwargs,
    250 ) -> DataFrame:
    251     kwargs["use_pandas_metadata"] = True
--> 253     path_or_handle, handles, filesystem = _get_path_or_handle(
    254         path,
    255         filesystem,
    256         storage_options=storage_options,
    257         mode="rb",
    258     )
    259     try:
    260         pa_table = self.api.parquet.read_table(
    261             path_or_handle,
    262             columns=columns,
   (...)    265             **kwargs,
    266         )

File ~/work/graphrag/graphrag/.venv/lib/python3.13/site-packages/pandas/io/parquet.py:141, in _get_path_or_handle(path, fs, storage_options, mode, is_dir)
    131 handles = None
    132 if (
    133     not fs
    134     and not is_dir
   (...)    139     # fsspec resources can also point to directories
    140     # this branch is used for example when reading from non-fsspec URLs
--> 141     handles = get_handle(
    142         path_or_handle, mode, is_text=False, storage_options=storage_options
    143     )
    144     fs = None
    145     path_or_handle = handles.handle

File ~/work/graphrag/graphrag/.venv/lib/python3.13/site-packages/pandas/io/common.py:939, in get_handle(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)
    930         handle = open(
    931             handle,
    932             ioargs.mode,
   (...)    935             newline="",
    936         )
    937     else:
    938         # Binary mode
--> 939         handle = open(handle, ioargs.mode)
    940     handles.append(handle)
    942 # Convert BytesIO or file objects passed with an encoding

FileNotFoundError: [Errno 2] No such file or directory: '<your project directory>/output/entities.parquet'

The response object contains GraphRAG's response, while the context object holds metadata about the querying process used to obtain the final response.

In [7]:

print(response)

print(response)

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Cell In[7], line 1
----> 1 print(response)

NameError: name 'response' is not defined

Digging into the context a bit more provides users with extremely granular information such as what sources of data (down to the level of text chunks) were ultimately retrieved and used as part of the context sent to the LLM model).

In [8]:

pprint(context)  # noqa: T203

pprint(context) # noqa: T203

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Cell In[8], line 1
----> 1 pprint(context)  # noqa: T203

NameError: name 'context' is not defined