Matt Holloway
2621dbefd9
* add sliding window for actions logs * refactor: fix sliding * remove trim content * only use up to 1mb of memory for logs * update to tail lines in second pass * add better memory usage calculation * increase window size to 5MB * update test * update vers * undo vers change * add incremental memory tracking * use ring buffer * remove unused ctx param * remove manual GC clear * fix cca feedback * extract ring buffer logic to new package * handle log content processing errors and use correct param for maxjobloglines * fix tailing * account for if tailLines exceeds window size * add profiling thats reusable * remove profiler testing * refactor profiler: introduce safeMemoryDelta for accurate memory delta calculations * linter fixes * Update pkg/buffer/buffer.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * use flag for maxJobLogLines * add param passing for context window size * refactor: rename contextWindowSize to contentWindowSize for consistency * fix: use tailLines if bigger but only if <= 5000 * fix: limit tailLines to a maximum of 500 for log content download * Update cmd/github-mcp-server/main.go Co-authored-by: Adam Holt <omgitsads@github.com> * Update cmd/github-mcp-server/main.go Co-authored-by: Adam Holt <omgitsads@github.com> * move profiler to internal/ * update actions test with new profiler location * fix: adjust buffer size limits * make line buffer 1028kb * fix mod path * change test to use same buffer size as normal use * improve test for non-sliding window implementation to not count empty lines * make test memory measurement more accurate * remove impossible conditional --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Adam Holt <omgitsads@github.com>
70 lines
2.0 KiB
Go
70 lines
2.0 KiB
Go
package buffer
|
|
|
|
import (
|
|
"bufio"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
)
|
|
|
|
// ProcessResponseAsRingBufferToEnd reads the body of an HTTP response line by line,
|
|
// storing only the last maxJobLogLines lines using a ring buffer (sliding window).
|
|
// This efficiently retains the most recent lines, overwriting older ones as needed.
|
|
//
|
|
// Parameters:
|
|
//
|
|
// httpResp: The HTTP response whose body will be read.
|
|
// maxJobLogLines: The maximum number of log lines to retain.
|
|
//
|
|
// Returns:
|
|
//
|
|
// string: The concatenated log lines (up to maxJobLogLines), separated by newlines.
|
|
// int: The total number of lines read from the response.
|
|
// *http.Response: The original HTTP response.
|
|
// error: Any error encountered during reading.
|
|
//
|
|
// The function uses a ring buffer to efficiently store only the last maxJobLogLines lines.
|
|
// If the response contains more lines than maxJobLogLines, only the most recent lines are kept.
|
|
func ProcessResponseAsRingBufferToEnd(httpResp *http.Response, maxJobLogLines int) (string, int, *http.Response, error) {
|
|
lines := make([]string, maxJobLogLines)
|
|
validLines := make([]bool, maxJobLogLines)
|
|
totalLines := 0
|
|
writeIndex := 0
|
|
|
|
scanner := bufio.NewScanner(httpResp.Body)
|
|
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
|
|
|
for scanner.Scan() {
|
|
line := scanner.Text()
|
|
totalLines++
|
|
|
|
lines[writeIndex] = line
|
|
validLines[writeIndex] = true
|
|
writeIndex = (writeIndex + 1) % maxJobLogLines
|
|
}
|
|
|
|
if err := scanner.Err(); err != nil {
|
|
return "", 0, httpResp, fmt.Errorf("failed to read log content: %w", err)
|
|
}
|
|
|
|
var result []string
|
|
linesInBuffer := totalLines
|
|
if linesInBuffer > maxJobLogLines {
|
|
linesInBuffer = maxJobLogLines
|
|
}
|
|
|
|
startIndex := 0
|
|
if totalLines > maxJobLogLines {
|
|
startIndex = writeIndex
|
|
}
|
|
|
|
for i := 0; i < linesInBuffer; i++ {
|
|
idx := (startIndex + i) % maxJobLogLines
|
|
if validLines[idx] {
|
|
result = append(result, lines[idx])
|
|
}
|
|
}
|
|
|
|
return strings.Join(result, "\n"), totalLines, httpResp, nil
|
|
}
|