Add rate limit checker to Discord integration

services/libs/integrations/src/integrations/discord/api/errorHandler.ts

@@ -32,6 +32,12 @@ export const handleDiscordError = (
 
     return new RateLimitError(rateLimitResetSeconds, url, err)
   }
+
+  if (err && err.response && err.response.status === 403) {
+    logger.warn('No access to resourse, ignoring it', { input, err })
+    return undefined
+  }
+
   logger.error(err, { input }, `Error while calling Discord API URL: ${url}`)
   return err
 }

services/libs/integrations/src/integrations/discord/api/getChannel.ts

@@ -2,12 +2,14 @@ import axios, { AxiosRequestConfig } from 'axios'
 import { handleDiscordError } from './errorHandler'
 import { DiscordApiChannel } from '../types'
 import { IProcessStreamContext } from '../../../types'
+import { getRateLimiter } from './handleRateLimit'
 
 export const getChannel = async (
   channelId: string,
   token: string,
   ctx: IProcessStreamContext,
 ): Promise<DiscordApiChannel> => {
+  const rateLimiter = getRateLimiter(ctx)
   const config: AxiosRequestConfig = {
     method: 'get',
     url: `https://discord.com/api/v10/channels/${channelId}`,
@@ -17,10 +19,14 @@ export const getChannel = async (
   }
 
   try {
+    await rateLimiter.checkRateLimit('getChannel')
+    await rateLimiter.incrementRateLimit()
     const response = await axios(config)
     return response.data
   } catch (err) {
     const newErr = handleDiscordError(err, config, { channelId }, ctx)
-    throw newErr
+    if (newErr) {
+      throw newErr
+    }
   }
 }

services/libs/integrations/src/integrations/discord/api/getChannels.ts

@@ -3,6 +3,8 @@ import { timeout } from '@crowd/common'
 import { DiscordApiChannel, DiscordGetChannelsInput, DiscordGetMessagesInput } from '../types'
 import getMessages from './getMessages'
 import { IProcessStreamContext } from '../../../types'
+import { getRateLimiter } from './handleRateLimit'
+import { handleDiscordError } from './errorHandler'
 
 /**
  * Try if a channel is readable
@@ -31,14 +33,19 @@ async function getChannels(
   ctx: IProcessStreamContext,
   tryChannels = true,
 ): Promise<DiscordApiChannel[]> {
+  const rateLimiter = getRateLimiter(ctx)
+
+  const config = {
+    method: 'get',
+    url: `https://discord.com/api/v10/guilds/${input.guildId}/channels?`,
+    headers: {
+      Authorization: input.token,
+    },
+  }
+
   try {
-    const config = {
-      method: 'get',
-      url: `https://discord.com/api/v10/guilds/${input.guildId}/channels?`,
-      headers: {
-        Authorization: input.token,
-      },
-    }
+    await rateLimiter.checkRateLimit('getChannels')
+    await rateLimiter.incrementRateLimit()
 
     const response = await axios(config)
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -69,8 +76,10 @@ async function getChannels(
 
     return result
   } catch (err) {
-    ctx.log.error({ err, input }, 'Error while getting channels from Discord')
-    throw err
+    const newErr = handleDiscordError(err, config, { input }, ctx)
+    if (newErr) {
+      throw newErr
+    }
   }
 }
 

services/libs/integrations/src/integrations/discord/api/getMember.ts

@@ -2,13 +2,16 @@ import axios, { AxiosRequestConfig } from 'axios'
 import { DiscordApiMember } from '../types'
 import { handleDiscordError } from './errorHandler'
 import { IProcessStreamContext } from '../../../types'
+import { getRateLimiter } from './handleRateLimit'
 
 export const getMember = async (
   guildId: string,
   userId: string,
   token: string,
   ctx: IProcessStreamContext,
 ): Promise<DiscordApiMember> => {
+  const rateLimiter = getRateLimiter(ctx)
+
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   const config: AxiosRequestConfig<any> = {
     method: 'get',
@@ -19,10 +22,14 @@ export const getMember = async (
   }
 
   try {
+    await rateLimiter.checkRateLimit('getMember')
+    await rateLimiter.incrementRateLimit()
     const response = await axios(config)
     return response.data
   } catch (err) {
     const newErr = handleDiscordError(err, config, { guildId, userId }, ctx)
-    throw newErr
+    if (newErr) {
+      throw newErr
+    }
   }
 }

services/libs/integrations/src/integrations/discord/api/getMembers.ts

@@ -1,11 +1,13 @@
 import axios from 'axios'
 import { DiscordApiMember, DiscordGetMembersInput, DiscordGetMembersOutput } from '../types'
 import { IProcessStreamContext } from '../../../types'
+import { getRateLimiter } from './handleRateLimit'
 
 async function getMembers(
   input: DiscordGetMembersInput,
   ctx: IProcessStreamContext,
 ): Promise<DiscordGetMembersOutput> {
+  const rateLimiter = getRateLimiter(ctx)
   try {
     let url = `https://discord.com/api/v10/guilds/${input.guildId}/members?limit=${input.perPage}`
     if (input.page !== undefined && input.page !== '') {
@@ -19,6 +21,8 @@ async function getMembers(
       },
     }
 
+    await rateLimiter.checkRateLimit('getMembers')
+    await rateLimiter.incrementRateLimit()
     const response = await axios(config)
     const records: DiscordApiMember[] = response.data
     const limit = parseInt(response.headers['x-ratelimit-remaining'], 10)

services/libs/integrations/src/integrations/discord/api/getMessage.ts

@@ -1,13 +1,16 @@
 import axios, { AxiosRequestConfig } from 'axios'
 import { handleDiscordError } from './errorHandler'
 import { IProcessStreamContext } from '../../../types'
+import { getRateLimiter } from './handleRateLimit'
 
 export const getMessage = async (
   channelId: string,
   messageId: string,
   token: string,
   ctx: IProcessStreamContext,
 ) => {
+  const rateLimiter = getRateLimiter(ctx)
+
   const config: AxiosRequestConfig = {
     method: 'get',
     url: `https://discord.com/api/v10/channels/${channelId}/messages/${messageId}`,
@@ -17,6 +20,8 @@ export const getMessage = async (
   }
 
   try {
+    await rateLimiter.checkRateLimit('getMessage')
+    await rateLimiter.incrementRateLimit()
     const response = await axios(config)
     return response.data
   } catch (err) {

services/libs/integrations/src/integrations/discord/api/getMessages.ts

@@ -1,12 +1,14 @@
 import axios from 'axios'
 import { DiscordApiMessage, DiscordParsedReponse, DiscordGetMessagesInput } from '../types'
 import { IProcessStreamContext } from '../../../types'
+import { getRateLimiter } from './handleRateLimit'
 
 async function getMessages(
   input: DiscordGetMessagesInput,
   ctx: IProcessStreamContext,
   showError = true,
 ): Promise<DiscordParsedReponse> {
+  const rateLimiter = getRateLimiter(ctx)
   try {
     let url = `https://discord.com/api/v10/channels/${input.channelId}/messages?limit=${input.perPage}`
     if (input.page !== undefined && input.page !== '') {
@@ -20,6 +22,8 @@ async function getMessages(
       },
     }
 
+    await rateLimiter.checkRateLimit('getMessages')
+    await rateLimiter.incrementRateLimit()
     const response = await axios(config)
     const records: DiscordApiMessage[] = response.data
 

services/libs/integrations/src/integrations/discord/api/getThreads.ts

@@ -1,25 +1,31 @@
 import axios from 'axios'
 import { DiscordApiChannel, DiscordGetChannelsInput } from '../types'
 import { IProcessStreamContext } from '../../../types'
+import { getRateLimiter } from './handleRateLimit'
+import { handleDiscordError } from './errorHandler'
 
 async function getThreads(
   input: DiscordGetChannelsInput,
   ctx: IProcessStreamContext,
 ): Promise<DiscordApiChannel[]> {
+  const rateLimiter = getRateLimiter(ctx)
+  const config = {
+    method: 'get',
+    url: `https://discord.com/api/v10/guilds/${input.guildId}/threads/active?`,
+    headers: {
+      Authorization: input.token,
+    },
+  }
   try {
-    const config = {
-      method: 'get',
-      url: `https://discord.com/api/v10/guilds/${input.guildId}/threads/active?`,
-      headers: {
-        Authorization: input.token,
-      },
-    }
-
+    await rateLimiter.checkRateLimit('getThreads')
+    await rateLimiter.incrementRateLimit()
     const response = await axios(config)
     return response.data.threads
   } catch (err) {
-    ctx.log.error({ err, input }, 'Error while getting threads from Discord')
-    throw err
+    const newErr = handleDiscordError(err, config, { input }, ctx)
+    if (newErr) {
+      throw newErr
+    }
   }
 }
 

services/libs/integrations/src/integrations/discord/api/handleRateLimit.ts

@@ -0,0 +1,9 @@
+import { IProcessStreamContext } from '../../../types'
+
+const DISCORD_RATE_LIMIT = 50
+const DISCORD_RATE_LIMIT_TIME = 1
+const REDIS_KEY = 'discord-request-count'
+
+export const getRateLimiter = (ctx: IProcessStreamContext) => {
+  return ctx.getRateLimiter(DISCORD_RATE_LIMIT, DISCORD_RATE_LIMIT_TIME, REDIS_KEY)
+}