Eric Allam
2b3ea692fe
* WIP * Run queue now works with the worker queue / master queue split * Acking should also cause the master queue to be processed * Convert run engine tests and run engine to use runQueue changes * Include the util files in the test tsconfig * coordinator target should be es2020 as well * providers target 2020 * Fix the triggerTask tests in the webapp * v4 now working with the new worker queues, and added the legacy master queue migration stuff * report worker queue lengths via opentelemetry metrics * Adding lock metrics * Release concurrency bucket metrics * • Updated RunQueue.removeEnvironmentQueuesFromMasterQueue() method signature to take runtimeEnvironmentId instead of masterQueue parameter • Added automatic master queue shard calculation using this.keys.masterQueueKeyForEnvironment(runtimeEnvironmentId, this.shardCount) • Updated RunEngine wrapper method to use new runtimeEnvironmentId parameter • Updated DeleteProjectService to call the method once per environment instead of once per master queue • Simplified API by encapsulating master queue sharding logic within RunQueue class * metrics now working, configure the run queue settings, additional metrics for run engine and redis-worker * Fix CodeRabbit suggestions * return undefined from dequeueFromWorkerQueue, not null * Remove message from worker queue in certain circumstances when acking * Update log * Ensure master queue consumers cannot stop from a processing error, and make the consumer interval configurable via an env var * Change how the run queue master queue consumers are disabled internally * Fixed tests * process the queue on nack * Fix more tests * Fix priority tests * Fixed dequeueing test
82 lines
2.6 KiB
TypeScript
82 lines
2.6 KiB
TypeScript
import { jumpHash } from "../src/v3/serverOnly/index.js";
|
|
|
|
describe("jumpHash", () => {
|
|
it("should hash a string to a number", () => {
|
|
expect(jumpHash("test", 10)).toBe(5);
|
|
});
|
|
|
|
it("should hash different strings to numbers in range", () => {
|
|
for (const key of ["a", "b", "c", "test", "trigger", "dev", "123", "!@#"]) {
|
|
for (const buckets of [1, 2, 5, 10, 100, 1000]) {
|
|
const result = jumpHash(key, buckets);
|
|
expect(result).toBeGreaterThanOrEqual(0);
|
|
expect(result).toBeLessThan(buckets);
|
|
}
|
|
}
|
|
});
|
|
|
|
it("should return 0 for any key if buckets is 1", () => {
|
|
expect(jumpHash("anything", 1)).toBe(0);
|
|
expect(jumpHash("", 1)).toBe(0);
|
|
});
|
|
|
|
it("should handle empty string key", () => {
|
|
expect(jumpHash("", 10)).toBeGreaterThanOrEqual(0);
|
|
expect(jumpHash("", 10)).toBeLessThan(10);
|
|
});
|
|
|
|
it("should distribute keys evenly across buckets", () => {
|
|
const buckets = 10;
|
|
const numKeys = 10000;
|
|
const counts = Array(buckets).fill(0);
|
|
for (let i = 0; i < numKeys; i++) {
|
|
const key = `key_${i}`;
|
|
const bucket = jumpHash(key, buckets);
|
|
counts[bucket]++;
|
|
}
|
|
const avg = numKeys / buckets;
|
|
// No bucket should have less than half or more than double the average
|
|
for (const count of counts) {
|
|
expect(count).toBeGreaterThanOrEqual(avg * 0.5);
|
|
expect(count).toBeLessThanOrEqual(avg * 2);
|
|
}
|
|
});
|
|
|
|
it("should have minimal movement when increasing buckets by 1", () => {
|
|
const numKeys = 1000;
|
|
const buckets = 50;
|
|
let moved = 0;
|
|
for (let i = 0; i < numKeys; i++) {
|
|
const key = `key_${i}`;
|
|
const bucket1 = jumpHash(key, buckets);
|
|
const bucket2 = jumpHash(key, buckets + 1);
|
|
if (bucket1 !== bucket2) moved++;
|
|
}
|
|
// For jump consistent hash, about 1/(buckets+1) of keys should move
|
|
const expectedMoved = numKeys / (buckets + 1);
|
|
expect(moved).toBeGreaterThanOrEqual(expectedMoved * 0.5);
|
|
expect(moved).toBeLessThanOrEqual(expectedMoved * 2);
|
|
});
|
|
|
|
it("should be deterministic for the same key and bucket count", () => {
|
|
for (let i = 0; i < 100; i++) {
|
|
const key = `key_${i}`;
|
|
const buckets = 20;
|
|
const result1 = jumpHash(key, buckets);
|
|
const result2 = jumpHash(key, buckets);
|
|
expect(result1).toBe(result2);
|
|
}
|
|
});
|
|
|
|
it("should always return a value in [0, buckets-1]", () => {
|
|
for (let i = 0; i < 100; i++) {
|
|
const key = `key_${i}`;
|
|
for (let buckets = 1; buckets < 50; buckets++) {
|
|
const result = jumpHash(key, buckets);
|
|
expect(result).toBeGreaterThanOrEqual(0);
|
|
expect(result).toBeLessThan(buckets);
|
|
}
|
|
}
|
|
});
|
|
});
|