Global search with dynamic community selection

In [1]:

# Copyright (c) 2024 Microsoft Corporation.
# Licensed under the MIT License.

# Copyright (c) 2024 Microsoft Corporation. # Licensed under the MIT License.

In [2]:

import os

import pandas as pd
from graphrag.query.indexer_adapters import (
    read_indexer_communities,
    read_indexer_entities,
    read_indexer_reports,
)
from graphrag.query.structured_search.global_search.community_context import (
    GlobalCommunityContext,
)
from graphrag.query.structured_search.global_search.search import GlobalSearch
from graphrag_llm.completion import create_completion
from graphrag_llm.config import ModelConfig

import os import pandas as pd from graphrag.query.indexer_adapters import ( read_indexer_communities, read_indexer_entities, read_indexer_reports, ) from graphrag.query.structured_search.global_search.community_context import ( GlobalCommunityContext, ) from graphrag.query.structured_search.global_search.search import GlobalSearch from graphrag_llm.completion import create_completion from graphrag_llm.config import ModelConfig

Global Search example¶

Global search method generates answers by searching over all AI-generated community reports in a map-reduce fashion. This is a resource-intensive method, but often gives good responses for questions that require an understanding of the dataset as a whole (e.g. What are the most significant values of the herbs mentioned in this notebook?).

LLM setup¶

In [3]:

from graphrag.tokenizer.get_tokenizer import get_tokenizer

api_key = os.environ["GRAPHRAG_API_KEY"]

config = ModelConfig(
    type="litellm",
    model_provider="openai",
    model="gpt-4.1",
    api_key=api_key,
)
model = create_completion(config)

tokenizer = get_tokenizer(config)

from graphrag.tokenizer.get_tokenizer import get_tokenizer api_key = os.environ["GRAPHRAG_API_KEY"] config = ModelConfig( type="litellm", model_provider="openai", model="gpt-4.1", api_key=api_key, ) model = create_completion(config) tokenizer = get_tokenizer(config)

Load community reports as context for global search¶

• Load all community reports in the community_reports table from the indexing engine, to be used as context data for global search.
• Load entities from the entities tables from the indexing engine, to be used for calculating community weights for context ranking. Note that this is optional (if no entities are provided, we will not calculate community weights and only use the rank attribute in the community reports table for context ranking)
• Load all communities in the communities table from the indexing engine, to be used to reconstruct the community graph hierarchy for dynamic community selection.

In [4]:

# parquet files generated from indexing pipeline
INPUT_DIR = "./inputs/operation dulce"
COMMUNITY_TABLE = "communities"
COMMUNITY_REPORT_TABLE = "community_reports"
ENTITY_TABLE = "entities"

# we don't fix a specific community level but instead use an agent to dynamicially
# search through all the community reports to check if they are relevant.
COMMUNITY_LEVEL = None

# parquet files generated from indexing pipeline INPUT_DIR = "./inputs/operation dulce" COMMUNITY_TABLE = "communities" COMMUNITY_REPORT_TABLE = "community_reports" ENTITY_TABLE = "entities" # we don't fix a specific community level but instead use an agent to dynamicially # search through all the community reports to check if they are relevant. COMMUNITY_LEVEL = None

In [5]:

community_df = pd.read_parquet(f"{INPUT_DIR}/{COMMUNITY_TABLE}.parquet")
entity_df = pd.read_parquet(f"{INPUT_DIR}/{ENTITY_TABLE}.parquet")
report_df = pd.read_parquet(f"{INPUT_DIR}/{COMMUNITY_REPORT_TABLE}.parquet")

communities = read_indexer_communities(community_df, report_df)
reports = read_indexer_reports(
    report_df,
    community_df,
    community_level=COMMUNITY_LEVEL,
    dynamic_community_selection=True,
)
entities = read_indexer_entities(
    entity_df, community_df, community_level=COMMUNITY_LEVEL
)

print(f"Total report count: {len(report_df)}")
print(
    f"Report count after filtering by community level {COMMUNITY_LEVEL}: {len(reports)}"
)

report_df.head()

community_df = pd.read_parquet(f"{INPUT_DIR}/{COMMUNITY_TABLE}.parquet") entity_df = pd.read_parquet(f"{INPUT_DIR}/{ENTITY_TABLE}.parquet") report_df = pd.read_parquet(f"{INPUT_DIR}/{COMMUNITY_REPORT_TABLE}.parquet") communities = read_indexer_communities(community_df, report_df) reports = read_indexer_reports( report_df, community_df, community_level=COMMUNITY_LEVEL, dynamic_community_selection=True, ) entities = read_indexer_entities( entity_df, community_df, community_level=COMMUNITY_LEVEL ) print(f"Total report count: {len(report_df)}") print( f"Report count after filtering by community level {COMMUNITY_LEVEL}: {len(reports)}" ) report_df.head()

Total report count: 10
Report count after filtering by community level None: 10

Out[5]:

	id	human_readable_id	community	level	parent	children	title	summary	full_content	rank	rating_explanation	findings	full_content_json	period	size
0	9785e64b429dacd70cce8238051f3438b387ebcd6920ef...	7	7	1	0	[]	Paranormal Military Squad and Operation Dulce	This community centers on the Paranormal Milit...	# Paranormal Military Squad and Operation Dulc...	8.5	The impact severity rating is high due to the ...	[{'explanation': 'The Paranormal Military Squa...	{\n    "title": "Paranormal Military Squad and...	2026-01-13	4
1	212c8ff1796bff731abe3d54f3216511ab15d5fb0d97eb...	8	8	1	0	[]	Paranormal Military Squad: Leadership and Scie...	This community centers on the Paranormal Milit...	# Paranormal Military Squad: Leadership and Sc...	8.0	The community poses a high impact due to its i...	[{'explanation': 'Taylor Cruz, often referred ...	{\n    "title": "Paranormal Military Squad: Le...	2026-01-13	3
2	dbff99eaacade8ac4337a9bde0f533313445f0643ab3d0...	9	9	1	0	[]	Paranormal Military Squad and Operation: Dulce...	This community centers around the Paranormal M...	# Paranormal Military Squad and Operation: Dul...	8.5	The community poses a high impact severity due...	[{'explanation': 'The military complex is the ...	{\n    "title": "Paranormal Military Squad and...	2026-01-13	3
3	b6e8ab1ac8ecc605bbec09ad78819b95ac08d62c5a9991...	0	0	0	-1	[7, 8, 9]	Paranormal Military Squad and Operation: Dulce	This community centers on the Paranormal Milit...	# Paranormal Military Squad and Operation: Dul...	8.5	The community poses a high impact due to its e...	[{'explanation': 'Taylor Cruz is the central a...	{\n    "title": "Paranormal Military Squad and...	2026-01-13	10
4	97038bee58306d13b709b615fb2b816cae0c11273a9e73...	1	1	0	-1	[]	Team of Agents Investigating Dulce Base	This community centers on a specialized team o...	# Team of Agents Investigating Dulce Base\n\nT...	7.5	The impact severity rating is high due to the ...	[{'explanation': 'The core of this community i...	{\n    "title": "Team of Agents Investigating ...	2026-01-13	5

Build global context with dynamic community selection¶

The goal of dynamic community selection is to reduce the number of community reports processed in the map-reduce operation. To do this, we take advantage of the hierarchical structure of the indexed dataset. We first ask the LLM to rate the relevance of each level 0 community to the user query, then traverse to its child nodes if the current community report is relevant.

You can still set a COMMUNITY_LEVEL to filter out lower-level community reports and apply dynamic community selection to the remaining reports.

Note that the dataset is quite small, consisting of only 20 communities across two levels (0 and 1). Dynamic community selection is more effective when there is a large amount of content to filter out.

In [6]:

context_builder = GlobalCommunityContext(
    community_reports=reports,
    communities=communities,
    entities=entities,  # default to None if you don't want to use community weights for ranking
    tokenizer=tokenizer,
    dynamic_community_selection=True,
    dynamic_community_selection_kwargs={
        "model": model,
        "tokenizer": tokenizer,
    },
)

context_builder = GlobalCommunityContext( community_reports=reports, communities=communities, entities=entities, # default to None if you don't want to use community weights for ranking tokenizer=tokenizer, dynamic_community_selection=True, dynamic_community_selection_kwargs={ "model": model, "tokenizer": tokenizer, }, )

Perform global search with dynamic community selection¶

In [7]:

context_builder_params = {
    "use_community_summary": False,  # False means using full community reports. True means using community short summaries.
    "shuffle_data": True,
    "include_community_rank": True,
    "min_community_rank": 0,
    "community_rank_name": "rank",
    "include_community_weight": True,
    "community_weight_name": "occurrence weight",
    "normalize_community_weight": True,
    "max_tokens": 12_000,  # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 5000)
    "context_name": "Reports",
}

map_llm_params = {
    "max_tokens": 1000,
    "temperature": 0.0,
}

reduce_llm_params = {
    "max_tokens": 2000,  # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 1000-1500)
    "temperature": 0.0,
}

context_builder_params = { "use_community_summary": False, # False means using full community reports. True means using community short summaries. "shuffle_data": True, "include_community_rank": True, "min_community_rank": 0, "community_rank_name": "rank", "include_community_weight": True, "community_weight_name": "occurrence weight", "normalize_community_weight": True, "max_tokens": 12_000, # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 5000) "context_name": "Reports", } map_llm_params = { "max_tokens": 1000, "temperature": 0.0, } reduce_llm_params = { "max_tokens": 2000, # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 1000-1500) "temperature": 0.0, }

In [8]:

search_engine = GlobalSearch(
    model=model,
    context_builder=context_builder,
    tokenizer=tokenizer,
    max_data_tokens=12_000,  # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 5000)
    map_llm_params=dict(map_llm_params),
    reduce_llm_params=dict(reduce_llm_params),
    allow_general_knowledge=False,  # set this to True will add instruction to encourage the LLM to incorporate general knowledge in the response, which may increase hallucinations, but could be useful in some use cases.
    json_mode=False,
    context_builder_params=context_builder_params,
    concurrent_coroutines=32,
    response_type="multiple paragraphs",  # free form text describing the response type and format, can be anything, e.g. prioritized list, single paragraph, multiple paragraphs, multiple-page report
)

search_engine = GlobalSearch( model=model, context_builder=context_builder, tokenizer=tokenizer, max_data_tokens=12_000, # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 5000) map_llm_params=dict(map_llm_params), reduce_llm_params=dict(reduce_llm_params), allow_general_knowledge=False, # set this to True will add instruction to encourage the LLM to incorporate general knowledge in the response, which may increase hallucinations, but could be useful in some use cases. json_mode=False, context_builder_params=context_builder_params, concurrent_coroutines=32, response_type="multiple paragraphs", # free form text describing the response type and format, can be anything, e.g. prioritized list, single paragraph, multiple paragraphs, multiple-page report )

In [9]:

result = await search_engine.search("What is operation dulce?")

print(result.response)

result = await search_engine.search("What is operation dulce?") print(result.response)

## Overview of Operation Dulce

Operation Dulce is a classified, high-stakes mission orchestrated by the Paranormal Military Squad. Its primary objective is to investigate the enigmatic Dulce Base, a secretive and technologically advanced underground facility located beneath Dulce. The operation is notable for its focus on uncovering hidden technologies, managing existential threats, and exploring anomalous phenomena within the base. The implications of this mission are considered profound, with potential consequences for security, intelligence, and possibly the future of humanity itself [Data: Reports (4, 3, 7, 8, 9, +more)].

## Core Objectives and Activities

At the heart of Operation Dulce is the retrieval and analysis of enigmatic alien technology. This technology was initially discovered at a crash site and subsequently transferred to the Dulce base for further study. The operation involves several key entities: the alien technology itself, the crash site where it was found, and the Dulce base, which serves as the hub for scientific investigation and containment. The process highlights the importance of technological transfer and investigation, with significant implications for scientific advancement and security [Data: Reports (2)].

## Key Personnel and Structure

Operation Dulce is marked by its complexity and the need for strict adherence to protocols, as well as adaptability among squad members. It serves as a convergence point for key personnel, including Sam Rivera, Alex Mercer, and Taylor Cruz. For those involved, the mission is regarded as both career-defining and potentially world-altering [Data: Reports (4, 3, 8, 1)].

The operation is supported by advanced technological infrastructure, secure communications, and a hierarchical command structure. The Paranormal Military Squad operates from a secure military complex, utilizing briefing rooms, equipment bays, and encrypted radio transmitters to maintain operational readiness and security throughout the mission [Data: Reports (5, 9, 7)].

## The Dulce Base: Central Node of Mystery

Dulce Base itself is the central node of mystery and potential danger within Operation Dulce. It is characterized by a tightly interwoven network of secrecy, advanced technology, and high-stakes investigation. The base is the focal point for both technical and security considerations, serving as the facility where the alien technology is housed and subjected to rigorous scientific analysis. Efforts at the base are directed toward understanding and potentially harnessing the capabilities of the recovered technology [Data: Reports (4, 3, 2)].

## Alien Technology and Its Implications

The alien technology recovered during Operation Dulce is described as a collection of enigmatic devices and circuitry of possible extraterrestrial origin. Its retrieval and subsequent study are the primary drivers of the operation. The technology is believed to possess the potential to revolutionize human knowledge of physics and reality, making its investigation a matter of utmost importance [Data: Reports (2)].

## Security, Containment, and Existential Threats

Given the extraordinary nature of the alien technology and its possible extraterrestrial origin, Operation Dulce involves significant security and containment considerations. The structured approach to managing these concerns includes transferring the technology to a secure facility like Dulce base to prevent unauthorized access or unintended consequences. The squad is tasked not only with uncovering secrets but also with preventing potential dangers from escalating, including the management of existential threats and unexplained phenomena [Data: Reports (4, 7, 8, 2)].

## The Crash Site and Technological Transfer

The crash site serves as the origin point for the technological transfer that underpins Operation Dulce. It is the location where the alien technology was initially discovered and retrieved. The subsequent transfer of this technology to Dulce base marks a critical juncture in the operation, enabling further analysis and study under controlled and secure conditions [Data: Reports (2)].

## Conclusion

In summary, Operation Dulce is a covert and highly significant mission focused on the investigation of the Dulce Base and the analysis of recovered alien technology. The operation is defined by its complexity, the high level of secrecy involved, and the potential for world-altering discoveries. It brings together advanced infrastructure, specialized personnel, and rigorous protocols to address both the opportunities and dangers presented by the enigmatic phenomena and technologies encountered [Data: Reports (4, 3, 7, 8, 9, 2, 5, 1, +more)].

In [10]:

# inspect the data used to build the context for the LLM responses
result.context_data["reports"]

# inspect the data used to build the context for the LLM responses result.context_data["reports"]

Out[10]:

	id	title	occurrence weight	content	rank
0	4	Dulce Base and Operation: Dulce Community	1.0	# Dulce Base and Operation: Dulce Community\n\...	9.0
1	3	Paranormal Military Squad and Operation: Dulce	1.0	# Paranormal Military Squad and Operation: Dul...	8.5
2	8	Paranormal Military Squad: Leadership and Scie...	1.0	# Paranormal Military Squad: Leadership and Sc...	8.0
3	7	Paranormal Military Squad and Operation Dulce	0.4	# Paranormal Military Squad and Operation Dulc...	8.5
4	5	Dulce Base Operational Command: Briefing Room ...	0.4	# Dulce Base Operational Command: Briefing Roo...	7.5
5	9	Paranormal Military Squad and Operation: Dulce...	0.2	# Paranormal Military Squad and Operation: Dul...	8.5
6	6	Dulce Base Mainframe Room Technology Network	0.2	# Dulce Base Mainframe Room Technology Network...	7.5
7	1	Team of Agents Investigating Dulce Base	0.2	# Team of Agents Investigating Dulce Base\n\nT...	7.5
8	2	Alien Technology Retrieval and Analysis at Dul...	0.2	# Alien Technology Retrieval and Analysis at D...	9.0

In [11]:

# inspect number of LLM calls and tokens in dynamic community selection
llm_calls = result.llm_calls_categories["build_context"]
prompt_tokens = result.prompt_tokens_categories["build_context"]
output_tokens = result.output_tokens_categories["build_context"]
print(
    f"Build context LLM calls: {llm_calls}. Prompt tokens: {prompt_tokens}. Output tokens: {output_tokens}."
)
# inspect number of LLM calls and tokens in map-reduce
llm_calls = result.llm_calls_categories["map"] + result.llm_calls_categories["reduce"]
prompt_tokens = (
    result.prompt_tokens_categories["map"] + result.prompt_tokens_categories["reduce"]
)
output_tokens = (
    result.output_tokens_categories["map"] + result.output_tokens_categories["reduce"]
)
print(
    f"Map-reduce LLM calls: {llm_calls}. Prompt tokens: {prompt_tokens}. Output tokens: {output_tokens}."
)

# inspect number of LLM calls and tokens in dynamic community selection llm_calls = result.llm_calls_categories["build_context"] prompt_tokens = result.prompt_tokens_categories["build_context"] output_tokens = result.output_tokens_categories["build_context"] print( f"Build context LLM calls: {llm_calls}. Prompt tokens: {prompt_tokens}. Output tokens: {output_tokens}." ) # inspect number of LLM calls and tokens in map-reduce llm_calls = result.llm_calls_categories["map"] + result.llm_calls_categories["reduce"] prompt_tokens = ( result.prompt_tokens_categories["map"] + result.prompt_tokens_categories["reduce"] ) output_tokens = ( result.output_tokens_categories["map"] + result.output_tokens_categories["reduce"] ) print( f"Map-reduce LLM calls: {llm_calls}. Prompt tokens: {prompt_tokens}. Output tokens: {output_tokens}." )

Build context LLM calls: 10. Prompt tokens: 11051. Output tokens: 1182.
Map-reduce LLM calls: 3. Prompt tokens: 11728. Output tokens: 1749.