Initial Commit
This commit is contained in:
882
node_modules/@grpc/grpc-js/src/load-balancer-outlier-detection.ts
generated
vendored
Normal file
882
node_modules/@grpc/grpc-js/src/load-balancer-outlier-detection.ts
generated
vendored
Normal file
@@ -0,0 +1,882 @@
|
||||
/*
|
||||
* Copyright 2022 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
import { ChannelOptions } from './channel-options';
|
||||
import { ConnectivityState } from './connectivity-state';
|
||||
import { LogVerbosity, Status } from './constants';
|
||||
import { durationToMs, isDuration, msToDuration } from './duration';
|
||||
import {
|
||||
ChannelControlHelper,
|
||||
createChildChannelControlHelper,
|
||||
registerLoadBalancerType,
|
||||
} from './experimental';
|
||||
import {
|
||||
getFirstUsableConfig,
|
||||
LoadBalancer,
|
||||
LoadBalancingConfig,
|
||||
validateLoadBalancingConfig,
|
||||
} from './load-balancer';
|
||||
import { ChildLoadBalancerHandler } from './load-balancer-child-handler';
|
||||
import { PickArgs, Picker, PickResult, PickResultType } from './picker';
|
||||
import {
|
||||
SubchannelAddress,
|
||||
subchannelAddressToString,
|
||||
} from './subchannel-address';
|
||||
import {
|
||||
BaseSubchannelWrapper,
|
||||
ConnectivityStateListener,
|
||||
SubchannelInterface,
|
||||
} from './subchannel-interface';
|
||||
import * as logging from './logging';
|
||||
|
||||
const TRACER_NAME = 'outlier_detection';
|
||||
|
||||
function trace(text: string): void {
|
||||
logging.trace(LogVerbosity.DEBUG, TRACER_NAME, text);
|
||||
}
|
||||
|
||||
const TYPE_NAME = 'outlier_detection';
|
||||
|
||||
const OUTLIER_DETECTION_ENABLED =
|
||||
(process.env.GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION ?? 'true') === 'true';
|
||||
|
||||
export interface SuccessRateEjectionConfig {
|
||||
readonly stdev_factor: number;
|
||||
readonly enforcement_percentage: number;
|
||||
readonly minimum_hosts: number;
|
||||
readonly request_volume: number;
|
||||
}
|
||||
|
||||
export interface FailurePercentageEjectionConfig {
|
||||
readonly threshold: number;
|
||||
readonly enforcement_percentage: number;
|
||||
readonly minimum_hosts: number;
|
||||
readonly request_volume: number;
|
||||
}
|
||||
|
||||
const defaultSuccessRateEjectionConfig: SuccessRateEjectionConfig = {
|
||||
stdev_factor: 1900,
|
||||
enforcement_percentage: 100,
|
||||
minimum_hosts: 5,
|
||||
request_volume: 100,
|
||||
};
|
||||
|
||||
const defaultFailurePercentageEjectionConfig: FailurePercentageEjectionConfig =
|
||||
{
|
||||
threshold: 85,
|
||||
enforcement_percentage: 100,
|
||||
minimum_hosts: 5,
|
||||
request_volume: 50,
|
||||
};
|
||||
|
||||
type TypeofValues =
|
||||
| 'object'
|
||||
| 'boolean'
|
||||
| 'function'
|
||||
| 'number'
|
||||
| 'string'
|
||||
| 'undefined';
|
||||
|
||||
function validateFieldType(
|
||||
obj: any,
|
||||
fieldName: string,
|
||||
expectedType: TypeofValues,
|
||||
objectName?: string
|
||||
) {
|
||||
if (fieldName in obj && typeof obj[fieldName] !== expectedType) {
|
||||
const fullFieldName = objectName ? `${objectName}.${fieldName}` : fieldName;
|
||||
throw new Error(
|
||||
`outlier detection config ${fullFieldName} parse error: expected ${expectedType}, got ${typeof obj[
|
||||
fieldName
|
||||
]}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function validatePositiveDuration(
|
||||
obj: any,
|
||||
fieldName: string,
|
||||
objectName?: string
|
||||
) {
|
||||
const fullFieldName = objectName ? `${objectName}.${fieldName}` : fieldName;
|
||||
if (fieldName in obj) {
|
||||
if (!isDuration(obj[fieldName])) {
|
||||
throw new Error(
|
||||
`outlier detection config ${fullFieldName} parse error: expected Duration, got ${typeof obj[
|
||||
fieldName
|
||||
]}`
|
||||
);
|
||||
}
|
||||
if (
|
||||
!(
|
||||
obj[fieldName].seconds >= 0 &&
|
||||
obj[fieldName].seconds <= 315_576_000_000 &&
|
||||
obj[fieldName].nanos >= 0 &&
|
||||
obj[fieldName].nanos <= 999_999_999
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
`outlier detection config ${fullFieldName} parse error: values out of range for non-negative Duaration`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function validatePercentage(obj: any, fieldName: string, objectName?: string) {
|
||||
const fullFieldName = objectName ? `${objectName}.${fieldName}` : fieldName;
|
||||
validateFieldType(obj, fieldName, 'number', objectName);
|
||||
if (fieldName in obj && !(obj[fieldName] >= 0 && obj[fieldName] <= 100)) {
|
||||
throw new Error(
|
||||
`outlier detection config ${fullFieldName} parse error: value out of range for percentage (0-100)`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export class OutlierDetectionLoadBalancingConfig
|
||||
implements LoadBalancingConfig
|
||||
{
|
||||
private readonly intervalMs: number;
|
||||
private readonly baseEjectionTimeMs: number;
|
||||
private readonly maxEjectionTimeMs: number;
|
||||
private readonly maxEjectionPercent: number;
|
||||
private readonly successRateEjection: SuccessRateEjectionConfig | null;
|
||||
private readonly failurePercentageEjection: FailurePercentageEjectionConfig | null;
|
||||
|
||||
constructor(
|
||||
intervalMs: number | null,
|
||||
baseEjectionTimeMs: number | null,
|
||||
maxEjectionTimeMs: number | null,
|
||||
maxEjectionPercent: number | null,
|
||||
successRateEjection: Partial<SuccessRateEjectionConfig> | null,
|
||||
failurePercentageEjection: Partial<FailurePercentageEjectionConfig> | null,
|
||||
private readonly childPolicy: LoadBalancingConfig[]
|
||||
) {
|
||||
if (
|
||||
childPolicy.length > 0 &&
|
||||
childPolicy[0].getLoadBalancerName() === 'pick_first'
|
||||
) {
|
||||
throw new Error(
|
||||
'outlier_detection LB policy cannot have a pick_first child policy'
|
||||
);
|
||||
}
|
||||
this.intervalMs = intervalMs ?? 10_000;
|
||||
this.baseEjectionTimeMs = baseEjectionTimeMs ?? 30_000;
|
||||
this.maxEjectionTimeMs = maxEjectionTimeMs ?? 300_000;
|
||||
this.maxEjectionPercent = maxEjectionPercent ?? 10;
|
||||
this.successRateEjection = successRateEjection
|
||||
? { ...defaultSuccessRateEjectionConfig, ...successRateEjection }
|
||||
: null;
|
||||
this.failurePercentageEjection = failurePercentageEjection
|
||||
? {
|
||||
...defaultFailurePercentageEjectionConfig,
|
||||
...failurePercentageEjection,
|
||||
}
|
||||
: null;
|
||||
}
|
||||
getLoadBalancerName(): string {
|
||||
return TYPE_NAME;
|
||||
}
|
||||
toJsonObject(): object {
|
||||
return {
|
||||
interval: msToDuration(this.intervalMs),
|
||||
base_ejection_time: msToDuration(this.baseEjectionTimeMs),
|
||||
max_ejection_time: msToDuration(this.maxEjectionTimeMs),
|
||||
max_ejection_percent: this.maxEjectionPercent,
|
||||
success_rate_ejection: this.successRateEjection,
|
||||
failure_percentage_ejection: this.failurePercentageEjection,
|
||||
child_policy: this.childPolicy.map(policy => policy.toJsonObject()),
|
||||
};
|
||||
}
|
||||
|
||||
getIntervalMs(): number {
|
||||
return this.intervalMs;
|
||||
}
|
||||
getBaseEjectionTimeMs(): number {
|
||||
return this.baseEjectionTimeMs;
|
||||
}
|
||||
getMaxEjectionTimeMs(): number {
|
||||
return this.maxEjectionTimeMs;
|
||||
}
|
||||
getMaxEjectionPercent(): number {
|
||||
return this.maxEjectionPercent;
|
||||
}
|
||||
getSuccessRateEjectionConfig(): SuccessRateEjectionConfig | null {
|
||||
return this.successRateEjection;
|
||||
}
|
||||
getFailurePercentageEjectionConfig(): FailurePercentageEjectionConfig | null {
|
||||
return this.failurePercentageEjection;
|
||||
}
|
||||
getChildPolicy(): LoadBalancingConfig[] {
|
||||
return this.childPolicy;
|
||||
}
|
||||
|
||||
copyWithChildPolicy(
|
||||
childPolicy: LoadBalancingConfig[]
|
||||
): OutlierDetectionLoadBalancingConfig {
|
||||
return new OutlierDetectionLoadBalancingConfig(
|
||||
this.intervalMs,
|
||||
this.baseEjectionTimeMs,
|
||||
this.maxEjectionTimeMs,
|
||||
this.maxEjectionPercent,
|
||||
this.successRateEjection,
|
||||
this.failurePercentageEjection,
|
||||
childPolicy
|
||||
);
|
||||
}
|
||||
|
||||
static createFromJson(obj: any): OutlierDetectionLoadBalancingConfig {
|
||||
validatePositiveDuration(obj, 'interval');
|
||||
validatePositiveDuration(obj, 'base_ejection_time');
|
||||
validatePositiveDuration(obj, 'max_ejection_time');
|
||||
validatePercentage(obj, 'max_ejection_percent');
|
||||
if ('success_rate_ejection' in obj) {
|
||||
if (typeof obj.success_rate_ejection !== 'object') {
|
||||
throw new Error(
|
||||
'outlier detection config success_rate_ejection must be an object'
|
||||
);
|
||||
}
|
||||
validateFieldType(
|
||||
obj.success_rate_ejection,
|
||||
'stdev_factor',
|
||||
'number',
|
||||
'success_rate_ejection'
|
||||
);
|
||||
validatePercentage(
|
||||
obj.success_rate_ejection,
|
||||
'enforcement_percentage',
|
||||
'success_rate_ejection'
|
||||
);
|
||||
validateFieldType(
|
||||
obj.success_rate_ejection,
|
||||
'minimum_hosts',
|
||||
'number',
|
||||
'success_rate_ejection'
|
||||
);
|
||||
validateFieldType(
|
||||
obj.success_rate_ejection,
|
||||
'request_volume',
|
||||
'number',
|
||||
'success_rate_ejection'
|
||||
);
|
||||
}
|
||||
if ('failure_percentage_ejection' in obj) {
|
||||
if (typeof obj.failure_percentage_ejection !== 'object') {
|
||||
throw new Error(
|
||||
'outlier detection config failure_percentage_ejection must be an object'
|
||||
);
|
||||
}
|
||||
validatePercentage(
|
||||
obj.failure_percentage_ejection,
|
||||
'threshold',
|
||||
'failure_percentage_ejection'
|
||||
);
|
||||
validatePercentage(
|
||||
obj.failure_percentage_ejection,
|
||||
'enforcement_percentage',
|
||||
'failure_percentage_ejection'
|
||||
);
|
||||
validateFieldType(
|
||||
obj.failure_percentage_ejection,
|
||||
'minimum_hosts',
|
||||
'number',
|
||||
'failure_percentage_ejection'
|
||||
);
|
||||
validateFieldType(
|
||||
obj.failure_percentage_ejection,
|
||||
'request_volume',
|
||||
'number',
|
||||
'failure_percentage_ejection'
|
||||
);
|
||||
}
|
||||
|
||||
return new OutlierDetectionLoadBalancingConfig(
|
||||
obj.interval ? durationToMs(obj.interval) : null,
|
||||
obj.base_ejection_time ? durationToMs(obj.base_ejection_time) : null,
|
||||
obj.max_ejection_time ? durationToMs(obj.max_ejection_time) : null,
|
||||
obj.max_ejection_percent ?? null,
|
||||
obj.success_rate_ejection,
|
||||
obj.failure_percentage_ejection,
|
||||
obj.child_policy.map(validateLoadBalancingConfig)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
class OutlierDetectionSubchannelWrapper
|
||||
extends BaseSubchannelWrapper
|
||||
implements SubchannelInterface
|
||||
{
|
||||
private childSubchannelState: ConnectivityState;
|
||||
private stateListeners: ConnectivityStateListener[] = [];
|
||||
private ejected = false;
|
||||
private refCount = 0;
|
||||
constructor(
|
||||
childSubchannel: SubchannelInterface,
|
||||
private mapEntry?: MapEntry
|
||||
) {
|
||||
super(childSubchannel);
|
||||
this.childSubchannelState = childSubchannel.getConnectivityState();
|
||||
childSubchannel.addConnectivityStateListener(
|
||||
(subchannel, previousState, newState, keepaliveTime) => {
|
||||
this.childSubchannelState = newState;
|
||||
if (!this.ejected) {
|
||||
for (const listener of this.stateListeners) {
|
||||
listener(this, previousState, newState, keepaliveTime);
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
getConnectivityState(): ConnectivityState {
|
||||
if (this.ejected) {
|
||||
return ConnectivityState.TRANSIENT_FAILURE;
|
||||
} else {
|
||||
return this.childSubchannelState;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a listener function to be called whenever the wrapper's
|
||||
* connectivity state changes.
|
||||
* @param listener
|
||||
*/
|
||||
addConnectivityStateListener(listener: ConnectivityStateListener) {
|
||||
this.stateListeners.push(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a listener previously added with `addConnectivityStateListener`
|
||||
* @param listener A reference to a function previously passed to
|
||||
* `addConnectivityStateListener`
|
||||
*/
|
||||
removeConnectivityStateListener(listener: ConnectivityStateListener) {
|
||||
const listenerIndex = this.stateListeners.indexOf(listener);
|
||||
if (listenerIndex > -1) {
|
||||
this.stateListeners.splice(listenerIndex, 1);
|
||||
}
|
||||
}
|
||||
|
||||
ref() {
|
||||
this.child.ref();
|
||||
this.refCount += 1;
|
||||
}
|
||||
|
||||
unref() {
|
||||
this.child.unref();
|
||||
this.refCount -= 1;
|
||||
if (this.refCount <= 0) {
|
||||
if (this.mapEntry) {
|
||||
const index = this.mapEntry.subchannelWrappers.indexOf(this);
|
||||
if (index >= 0) {
|
||||
this.mapEntry.subchannelWrappers.splice(index, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
eject() {
|
||||
this.ejected = true;
|
||||
for (const listener of this.stateListeners) {
|
||||
listener(
|
||||
this,
|
||||
this.childSubchannelState,
|
||||
ConnectivityState.TRANSIENT_FAILURE,
|
||||
-1
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
uneject() {
|
||||
this.ejected = false;
|
||||
for (const listener of this.stateListeners) {
|
||||
listener(
|
||||
this,
|
||||
ConnectivityState.TRANSIENT_FAILURE,
|
||||
this.childSubchannelState,
|
||||
-1
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
getMapEntry(): MapEntry | undefined {
|
||||
return this.mapEntry;
|
||||
}
|
||||
|
||||
getWrappedSubchannel(): SubchannelInterface {
|
||||
return this.child;
|
||||
}
|
||||
}
|
||||
|
||||
interface CallCountBucket {
|
||||
success: number;
|
||||
failure: number;
|
||||
}
|
||||
|
||||
function createEmptyBucket(): CallCountBucket {
|
||||
return {
|
||||
success: 0,
|
||||
failure: 0,
|
||||
};
|
||||
}
|
||||
|
||||
class CallCounter {
|
||||
private activeBucket: CallCountBucket = createEmptyBucket();
|
||||
private inactiveBucket: CallCountBucket = createEmptyBucket();
|
||||
addSuccess() {
|
||||
this.activeBucket.success += 1;
|
||||
}
|
||||
addFailure() {
|
||||
this.activeBucket.failure += 1;
|
||||
}
|
||||
switchBuckets() {
|
||||
this.inactiveBucket = this.activeBucket;
|
||||
this.activeBucket = createEmptyBucket();
|
||||
}
|
||||
getLastSuccesses() {
|
||||
return this.inactiveBucket.success;
|
||||
}
|
||||
getLastFailures() {
|
||||
return this.inactiveBucket.failure;
|
||||
}
|
||||
}
|
||||
|
||||
interface MapEntry {
|
||||
counter: CallCounter;
|
||||
currentEjectionTimestamp: Date | null;
|
||||
ejectionTimeMultiplier: number;
|
||||
subchannelWrappers: OutlierDetectionSubchannelWrapper[];
|
||||
}
|
||||
|
||||
class OutlierDetectionPicker implements Picker {
|
||||
constructor(private wrappedPicker: Picker, private countCalls: boolean) {}
|
||||
pick(pickArgs: PickArgs): PickResult {
|
||||
const wrappedPick = this.wrappedPicker.pick(pickArgs);
|
||||
if (wrappedPick.pickResultType === PickResultType.COMPLETE) {
|
||||
const subchannelWrapper =
|
||||
wrappedPick.subchannel as OutlierDetectionSubchannelWrapper;
|
||||
const mapEntry = subchannelWrapper.getMapEntry();
|
||||
if (mapEntry) {
|
||||
let onCallEnded = wrappedPick.onCallEnded;
|
||||
if (this.countCalls) {
|
||||
onCallEnded = statusCode => {
|
||||
if (statusCode === Status.OK) {
|
||||
mapEntry.counter.addSuccess();
|
||||
} else {
|
||||
mapEntry.counter.addFailure();
|
||||
}
|
||||
wrappedPick.onCallEnded?.(statusCode);
|
||||
};
|
||||
}
|
||||
return {
|
||||
...wrappedPick,
|
||||
subchannel: subchannelWrapper.getWrappedSubchannel(),
|
||||
onCallEnded: onCallEnded,
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
...wrappedPick,
|
||||
subchannel: subchannelWrapper.getWrappedSubchannel(),
|
||||
};
|
||||
}
|
||||
} else {
|
||||
return wrappedPick;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class OutlierDetectionLoadBalancer implements LoadBalancer {
|
||||
private childBalancer: ChildLoadBalancerHandler;
|
||||
private addressMap: Map<string, MapEntry> = new Map<string, MapEntry>();
|
||||
private latestConfig: OutlierDetectionLoadBalancingConfig | null = null;
|
||||
private ejectionTimer: NodeJS.Timeout;
|
||||
private timerStartTime: Date | null = null;
|
||||
|
||||
constructor(channelControlHelper: ChannelControlHelper) {
|
||||
this.childBalancer = new ChildLoadBalancerHandler(
|
||||
createChildChannelControlHelper(channelControlHelper, {
|
||||
createSubchannel: (
|
||||
subchannelAddress: SubchannelAddress,
|
||||
subchannelArgs: ChannelOptions
|
||||
) => {
|
||||
const originalSubchannel = channelControlHelper.createSubchannel(
|
||||
subchannelAddress,
|
||||
subchannelArgs
|
||||
);
|
||||
const mapEntry = this.addressMap.get(
|
||||
subchannelAddressToString(subchannelAddress)
|
||||
);
|
||||
const subchannelWrapper = new OutlierDetectionSubchannelWrapper(
|
||||
originalSubchannel,
|
||||
mapEntry
|
||||
);
|
||||
if (mapEntry?.currentEjectionTimestamp !== null) {
|
||||
// If the address is ejected, propagate that to the new subchannel wrapper
|
||||
subchannelWrapper.eject();
|
||||
}
|
||||
mapEntry?.subchannelWrappers.push(subchannelWrapper);
|
||||
return subchannelWrapper;
|
||||
},
|
||||
updateState: (connectivityState: ConnectivityState, picker: Picker) => {
|
||||
if (connectivityState === ConnectivityState.READY) {
|
||||
channelControlHelper.updateState(
|
||||
connectivityState,
|
||||
new OutlierDetectionPicker(picker, this.isCountingEnabled())
|
||||
);
|
||||
} else {
|
||||
channelControlHelper.updateState(connectivityState, picker);
|
||||
}
|
||||
},
|
||||
})
|
||||
);
|
||||
this.ejectionTimer = setInterval(() => {}, 0);
|
||||
clearInterval(this.ejectionTimer);
|
||||
}
|
||||
|
||||
private isCountingEnabled(): boolean {
|
||||
return (
|
||||
this.latestConfig !== null &&
|
||||
(this.latestConfig.getSuccessRateEjectionConfig() !== null ||
|
||||
this.latestConfig.getFailurePercentageEjectionConfig() !== null)
|
||||
);
|
||||
}
|
||||
|
||||
private getCurrentEjectionPercent() {
|
||||
let ejectionCount = 0;
|
||||
for (const mapEntry of this.addressMap.values()) {
|
||||
if (mapEntry.currentEjectionTimestamp !== null) {
|
||||
ejectionCount += 1;
|
||||
}
|
||||
}
|
||||
return (ejectionCount * 100) / this.addressMap.size;
|
||||
}
|
||||
|
||||
private runSuccessRateCheck(ejectionTimestamp: Date) {
|
||||
if (!this.latestConfig) {
|
||||
return;
|
||||
}
|
||||
const successRateConfig = this.latestConfig.getSuccessRateEjectionConfig();
|
||||
if (!successRateConfig) {
|
||||
return;
|
||||
}
|
||||
trace('Running success rate check');
|
||||
// Step 1
|
||||
const targetRequestVolume = successRateConfig.request_volume;
|
||||
let addresesWithTargetVolume = 0;
|
||||
const successRates: number[] = [];
|
||||
for (const [address, mapEntry] of this.addressMap) {
|
||||
const successes = mapEntry.counter.getLastSuccesses();
|
||||
const failures = mapEntry.counter.getLastFailures();
|
||||
trace(
|
||||
'Stats for ' +
|
||||
address +
|
||||
': successes=' +
|
||||
successes +
|
||||
' failures=' +
|
||||
failures +
|
||||
' targetRequestVolume=' +
|
||||
targetRequestVolume
|
||||
);
|
||||
if (successes + failures >= targetRequestVolume) {
|
||||
addresesWithTargetVolume += 1;
|
||||
successRates.push(successes / (successes + failures));
|
||||
}
|
||||
}
|
||||
trace(
|
||||
'Found ' +
|
||||
addresesWithTargetVolume +
|
||||
' success rate candidates; currentEjectionPercent=' +
|
||||
this.getCurrentEjectionPercent() +
|
||||
' successRates=[' +
|
||||
successRates +
|
||||
']'
|
||||
);
|
||||
if (addresesWithTargetVolume < successRateConfig.minimum_hosts) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Step 2
|
||||
const successRateMean =
|
||||
successRates.reduce((a, b) => a + b) / successRates.length;
|
||||
let successRateDeviationSum = 0;
|
||||
for (const rate of successRates) {
|
||||
const deviation = rate - successRateMean;
|
||||
successRateDeviationSum += deviation * deviation;
|
||||
}
|
||||
const successRateVariance = successRateDeviationSum / successRates.length;
|
||||
const successRateStdev = Math.sqrt(successRateVariance);
|
||||
const ejectionThreshold =
|
||||
successRateMean -
|
||||
successRateStdev * (successRateConfig.stdev_factor / 1000);
|
||||
trace(
|
||||
'stdev=' + successRateStdev + ' ejectionThreshold=' + ejectionThreshold
|
||||
);
|
||||
|
||||
// Step 3
|
||||
for (const [address, mapEntry] of this.addressMap.entries()) {
|
||||
// Step 3.i
|
||||
if (
|
||||
this.getCurrentEjectionPercent() >=
|
||||
this.latestConfig.getMaxEjectionPercent()
|
||||
) {
|
||||
break;
|
||||
}
|
||||
// Step 3.ii
|
||||
const successes = mapEntry.counter.getLastSuccesses();
|
||||
const failures = mapEntry.counter.getLastFailures();
|
||||
if (successes + failures < targetRequestVolume) {
|
||||
continue;
|
||||
}
|
||||
// Step 3.iii
|
||||
const successRate = successes / (successes + failures);
|
||||
trace('Checking candidate ' + address + ' successRate=' + successRate);
|
||||
if (successRate < ejectionThreshold) {
|
||||
const randomNumber = Math.random() * 100;
|
||||
trace(
|
||||
'Candidate ' +
|
||||
address +
|
||||
' randomNumber=' +
|
||||
randomNumber +
|
||||
' enforcement_percentage=' +
|
||||
successRateConfig.enforcement_percentage
|
||||
);
|
||||
if (randomNumber < successRateConfig.enforcement_percentage) {
|
||||
trace('Ejecting candidate ' + address);
|
||||
this.eject(mapEntry, ejectionTimestamp);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private runFailurePercentageCheck(ejectionTimestamp: Date) {
|
||||
if (!this.latestConfig) {
|
||||
return;
|
||||
}
|
||||
const failurePercentageConfig =
|
||||
this.latestConfig.getFailurePercentageEjectionConfig();
|
||||
if (!failurePercentageConfig) {
|
||||
return;
|
||||
}
|
||||
trace(
|
||||
'Running failure percentage check. threshold=' +
|
||||
failurePercentageConfig.threshold +
|
||||
' request volume threshold=' +
|
||||
failurePercentageConfig.request_volume
|
||||
);
|
||||
// Step 1
|
||||
let addressesWithTargetVolume = 0;
|
||||
for (const mapEntry of this.addressMap.values()) {
|
||||
const successes = mapEntry.counter.getLastSuccesses();
|
||||
const failures = mapEntry.counter.getLastFailures();
|
||||
if (successes + failures >= failurePercentageConfig.request_volume) {
|
||||
addressesWithTargetVolume += 1;
|
||||
}
|
||||
}
|
||||
if (addressesWithTargetVolume < failurePercentageConfig.minimum_hosts) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Step 2
|
||||
for (const [address, mapEntry] of this.addressMap.entries()) {
|
||||
// Step 2.i
|
||||
if (
|
||||
this.getCurrentEjectionPercent() >=
|
||||
this.latestConfig.getMaxEjectionPercent()
|
||||
) {
|
||||
break;
|
||||
}
|
||||
// Step 2.ii
|
||||
const successes = mapEntry.counter.getLastSuccesses();
|
||||
const failures = mapEntry.counter.getLastFailures();
|
||||
trace('Candidate successes=' + successes + ' failures=' + failures);
|
||||
if (successes + failures < failurePercentageConfig.request_volume) {
|
||||
continue;
|
||||
}
|
||||
// Step 2.iii
|
||||
const failurePercentage = (failures * 100) / (failures + successes);
|
||||
if (failurePercentage > failurePercentageConfig.threshold) {
|
||||
const randomNumber = Math.random() * 100;
|
||||
trace(
|
||||
'Candidate ' +
|
||||
address +
|
||||
' randomNumber=' +
|
||||
randomNumber +
|
||||
' enforcement_percentage=' +
|
||||
failurePercentageConfig.enforcement_percentage
|
||||
);
|
||||
if (randomNumber < failurePercentageConfig.enforcement_percentage) {
|
||||
trace('Ejecting candidate ' + address);
|
||||
this.eject(mapEntry, ejectionTimestamp);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private eject(mapEntry: MapEntry, ejectionTimestamp: Date) {
|
||||
mapEntry.currentEjectionTimestamp = new Date();
|
||||
mapEntry.ejectionTimeMultiplier += 1;
|
||||
for (const subchannelWrapper of mapEntry.subchannelWrappers) {
|
||||
subchannelWrapper.eject();
|
||||
}
|
||||
}
|
||||
|
||||
private uneject(mapEntry: MapEntry) {
|
||||
mapEntry.currentEjectionTimestamp = null;
|
||||
for (const subchannelWrapper of mapEntry.subchannelWrappers) {
|
||||
subchannelWrapper.uneject();
|
||||
}
|
||||
}
|
||||
|
||||
private switchAllBuckets() {
|
||||
for (const mapEntry of this.addressMap.values()) {
|
||||
mapEntry.counter.switchBuckets();
|
||||
}
|
||||
}
|
||||
|
||||
private startTimer(delayMs: number) {
|
||||
this.ejectionTimer = setTimeout(() => this.runChecks(), delayMs);
|
||||
this.ejectionTimer.unref?.();
|
||||
}
|
||||
|
||||
private runChecks() {
|
||||
const ejectionTimestamp = new Date();
|
||||
trace('Ejection timer running');
|
||||
|
||||
this.switchAllBuckets();
|
||||
|
||||
if (!this.latestConfig) {
|
||||
return;
|
||||
}
|
||||
this.timerStartTime = ejectionTimestamp;
|
||||
this.startTimer(this.latestConfig.getIntervalMs());
|
||||
|
||||
this.runSuccessRateCheck(ejectionTimestamp);
|
||||
this.runFailurePercentageCheck(ejectionTimestamp);
|
||||
|
||||
for (const [address, mapEntry] of this.addressMap.entries()) {
|
||||
if (mapEntry.currentEjectionTimestamp === null) {
|
||||
if (mapEntry.ejectionTimeMultiplier > 0) {
|
||||
mapEntry.ejectionTimeMultiplier -= 1;
|
||||
}
|
||||
} else {
|
||||
const baseEjectionTimeMs = this.latestConfig.getBaseEjectionTimeMs();
|
||||
const maxEjectionTimeMs = this.latestConfig.getMaxEjectionTimeMs();
|
||||
const returnTime = new Date(
|
||||
mapEntry.currentEjectionTimestamp.getTime()
|
||||
);
|
||||
returnTime.setMilliseconds(
|
||||
returnTime.getMilliseconds() +
|
||||
Math.min(
|
||||
baseEjectionTimeMs * mapEntry.ejectionTimeMultiplier,
|
||||
Math.max(baseEjectionTimeMs, maxEjectionTimeMs)
|
||||
)
|
||||
);
|
||||
if (returnTime < new Date()) {
|
||||
trace('Unejecting ' + address);
|
||||
this.uneject(mapEntry);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
updateAddressList(
|
||||
addressList: SubchannelAddress[],
|
||||
lbConfig: LoadBalancingConfig,
|
||||
attributes: { [key: string]: unknown }
|
||||
): void {
|
||||
if (!(lbConfig instanceof OutlierDetectionLoadBalancingConfig)) {
|
||||
return;
|
||||
}
|
||||
const subchannelAddresses = new Set<string>();
|
||||
for (const address of addressList) {
|
||||
subchannelAddresses.add(subchannelAddressToString(address));
|
||||
}
|
||||
for (const address of subchannelAddresses) {
|
||||
if (!this.addressMap.has(address)) {
|
||||
trace('Adding map entry for ' + address);
|
||||
this.addressMap.set(address, {
|
||||
counter: new CallCounter(),
|
||||
currentEjectionTimestamp: null,
|
||||
ejectionTimeMultiplier: 0,
|
||||
subchannelWrappers: [],
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const key of this.addressMap.keys()) {
|
||||
if (!subchannelAddresses.has(key)) {
|
||||
trace('Removing map entry for ' + key);
|
||||
this.addressMap.delete(key);
|
||||
}
|
||||
}
|
||||
const childPolicy: LoadBalancingConfig = getFirstUsableConfig(
|
||||
lbConfig.getChildPolicy(),
|
||||
true
|
||||
);
|
||||
this.childBalancer.updateAddressList(addressList, childPolicy, attributes);
|
||||
|
||||
if (
|
||||
lbConfig.getSuccessRateEjectionConfig() ||
|
||||
lbConfig.getFailurePercentageEjectionConfig()
|
||||
) {
|
||||
if (this.timerStartTime) {
|
||||
trace('Previous timer existed. Replacing timer');
|
||||
clearTimeout(this.ejectionTimer);
|
||||
const remainingDelay =
|
||||
lbConfig.getIntervalMs() -
|
||||
(new Date().getTime() - this.timerStartTime.getTime());
|
||||
this.startTimer(remainingDelay);
|
||||
} else {
|
||||
trace('Starting new timer');
|
||||
this.timerStartTime = new Date();
|
||||
this.startTimer(lbConfig.getIntervalMs());
|
||||
this.switchAllBuckets();
|
||||
}
|
||||
} else {
|
||||
trace('Counting disabled. Cancelling timer.');
|
||||
this.timerStartTime = null;
|
||||
clearTimeout(this.ejectionTimer);
|
||||
for (const mapEntry of this.addressMap.values()) {
|
||||
this.uneject(mapEntry);
|
||||
mapEntry.ejectionTimeMultiplier = 0;
|
||||
}
|
||||
}
|
||||
|
||||
this.latestConfig = lbConfig;
|
||||
}
|
||||
exitIdle(): void {
|
||||
this.childBalancer.exitIdle();
|
||||
}
|
||||
resetBackoff(): void {
|
||||
this.childBalancer.resetBackoff();
|
||||
}
|
||||
destroy(): void {
|
||||
clearTimeout(this.ejectionTimer);
|
||||
this.childBalancer.destroy();
|
||||
}
|
||||
getTypeName(): string {
|
||||
return TYPE_NAME;
|
||||
}
|
||||
}
|
||||
|
||||
export function setup() {
|
||||
if (OUTLIER_DETECTION_ENABLED) {
|
||||
registerLoadBalancerType(
|
||||
TYPE_NAME,
|
||||
OutlierDetectionLoadBalancer,
|
||||
OutlierDetectionLoadBalancingConfig
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user