Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions apps/bot/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,12 @@
"@craig/metrics": "workspace:*",
"@discordjs/opus": "^0.10.0",
"@henrygd/semaphore": "catalog:",
"@projectdysnomia/dysnomia": "github:projectdysnomia/dysnomia#00d8982d81d365c2cdb2ef5dac59af07977a65d7",
"@projectdysnomia/dysnomia": "github:Snazzah/dysnomia#da3811d70a678bf58480bf99b14fffeb55161427",
"@sentry/node": "^11.6.0",
"@snazzah/davey": "^0.1.12",
"@snazzah/davey": "^0.2.0-beta.1",
"@snazzah/emoji-sync": "^0.1.2",
"common-tags": "catalog:",
"cron": "catalog:",
"erlpack": "^0.1.4",
"fastify": "catalog:",
"fastq": "^1.19.1",
"i18next": "^25.7.2",
Expand Down
14 changes: 13 additions & 1 deletion apps/bot/src/modules/metrics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ export default class MetricsModule extends BotModule {
commandsRan: 0,
gatewayEventsReceived: 0,
commands: {} as Record<string, number>,
voiceServersConnected: {} as Record<string, number>
voiceServersConnected: {} as Record<string, number>,
recordingErrors: {} as Record<string, number>
};
private readonly handleRawWS = () => {
this.stats.gatewayEventsReceived++;
Expand Down Expand Up @@ -37,12 +38,23 @@ export default class MetricsModule extends BotModule {
else this.stats.voiceServersConnected[region]++;
}

onRecordingError(type: string, code = 'none') {
const key = `${type}:${code}`;
if (!this.stats.recordingErrors[key]) this.stats.recordingErrors[key] = 1;
else this.stats.recordingErrors[key]++;
}

collect(name: keyof typeof this.stats) {
if (name === 'commands') {
const commands = this.stats.commands;
this.stats.commands = {};
return commands;
}
if (name === 'recordingErrors') {
const stats = this.stats.recordingErrors;
this.stats.recordingErrors = {};
return stats;
}
if (name === 'voiceServersConnected') {
const stats = this.stats.voiceServersConnected;
this.stats.voiceServersConnected = {};
Expand Down
13 changes: 13 additions & 0 deletions apps/bot/src/modules/recorder/recording.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,12 @@ const MAX_ACTIVITY_LOGS = 10;
const MAX_LATENCY_WARNING = 500;
const opusSilenceFrame = Buffer.from([0xf8, 0xff, 0xfe]);
const ENCRYPTION_RECOVERY_STOP_THRESHOLD = 5;

// Only pass through Node-style error codes (e.g. ENOTFOUND) to keep metric label cardinality bounded
function getErrorCode(err: unknown) {
const code = (err as any)?.code;
return typeof code === 'string' && /^E[A-Z0-9_]{1,40}$/.test(code) ? code : 'unknown';
}
const BAD_MESSAGE_CODES = [
404,
10003, // Unknown channel
Expand Down Expand Up @@ -202,6 +208,7 @@ export default class Recording {
await this.connect();
} catch (e) {
this.recorder.traceVoiceTimeout(this.connection);
this.recorder.metrics.onRecordingError('connect_failed', getErrorCode(e));
this.recorder.logger.error(
`Failed to connect to ${this.channel.name} (${this.channel.id}) in ${this.channel.guild.name} (${this.channel.guild.id}) by ${this.user.username}#${this.user.discriminator} (${this.user.id})`,
e
Expand Down Expand Up @@ -503,6 +510,7 @@ export default class Recording {
}
if (this.state !== RecordingState.RECORDING) {
this.pushToActivity(this.t('recording.panel.reconnect_fail'), false);
this.recorder.metrics.onRecordingError('reconnect_failed');
this.sendWarning(this.t('recording.failed_to_reconnect'), false);
this.recorder.logger.warn(`Recording ${this.id} could not properly reconnect`);
try {
Expand Down Expand Up @@ -590,6 +598,7 @@ export default class Recording {
if (!this.active || this.encryptionStopIssued || !message.startsWith('Invalidating transition ')) return;

this.encryptionRecoveryAttempts++;
this.recorder.metrics.onRecordingError('decryption_tolerance_exceeded');

const endpoint = this.connection?.endpoint;
const context = {
Expand All @@ -608,6 +617,7 @@ export default class Recording {
if (this.encryptionRecoveryAttempts < ENCRYPTION_RECOVERY_STOP_THRESHOLD) return;

this.encryptionStopIssued = true;
this.recorder.metrics.onRecordingError('e2ee_stopped');
const stopWarning = this.t('recording.e2ee_error');
this.stateDescription = `⚠️ ${stopWarning}`;
await this.sendWarning(stopWarning, false);
Expand All @@ -617,6 +627,7 @@ export default class Recording {
async onConnectionError(err: any) {
this.writeToLog(`Error: ${err}`, 'connection');
this.recorder.logger.error(`Error in connection for recording ${this.id}`, err);
this.recorder.metrics.onRecordingError('connection', getErrorCode(err));

// The voice server's hostname couldn't be resolved
if (err?.code === 'ENOTFOUND' && this.active && this.state === RecordingState.RECORDING) {
Expand All @@ -642,6 +653,7 @@ export default class Recording {
rtcWorkerVersion: this.rtcWorkerVersion
});
if (err) {
this.recorder.metrics.onRecordingError('disconnect', String(this.recorder.parseVoiceDisconnectCode(err) ?? 'unknown'));
this.state = RecordingState.RECONNECTING;
if (err.message.startsWith('4006')) this.pushToActivity(this.t('recording.panel.discord_requested_reconnect'));
else this.pushToActivity(this.t('recording.panel.disconnected_from_error'));
Expand Down Expand Up @@ -697,6 +709,7 @@ export default class Recording {
this.encodeChunk(user, user.track, packetNo, chunk!);
packetNo += 2;
} catch (ex) {
this.recorder.metrics.onRecordingError('encode_failed');
this.recorder.logger.error(`Failed to encode packet ${packetNo} for user ${user.id}`, ex);
}
}
Expand Down
23 changes: 23 additions & 0 deletions apps/bot/src/sharding/modules/metrics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,29 @@ export default class MetricsModule extends ShardManagerModule {
}
});

new Counter({
name: 'craig_bot_recording_errors_total',
help: 'Counter for recording errors by type and code',
labelNames: ['type', 'code'],
async collect() {
try {
await timeout(
Promise.all(
Array.from(manager.shards.values()).map(async (shard) => {
if (shard.status === 'ready' || shard.status === 'resuming') {
const result = (await shard.eval('this.metrics.collect("recordingErrors")')) as Record<string, number>;
for (const key in result) {
const [type, code] = key.split(':');
this.inc({ type, code }, result[key]);
}
}
})
)
);
} catch {}
}
});

new Gauge({
name: 'craig_bot_shard_latency_milliseconds',
help: 'Gauge for millisecond latency per shard',
Expand Down
2 changes: 1 addition & 1 deletion apps/botctl/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
},
"dependencies": {
"@craig/logger": "workspace:*",
"@projectdysnomia/dysnomia": "github:projectdysnomia/dysnomia#00d8982d81d365c2cdb2ef5dac59af07977a65d7",
"@projectdysnomia/dysnomia": "github:Snazzah/dysnomia#da3811d70a678bf58480bf99b14fffeb55161427",
"common-tags": "catalog:",
"slash-create": "^6.8.1"
},
Expand Down
Loading
Loading