refactor: use queue/jobs to load entire matchlist ⚡

server/.adonisrc.json

@@ -4,7 +4,8 @@
     "./commands",
     "@adonisjs/core/build/commands/index.js",
     "@adonisjs/repl/build/commands",
-    "@adonisjs/lucid/build/commands"
+    "@adonisjs/lucid/build/commands",
+    "@rocketseat/adonis-bull/build/commands"
   ],
   "exceptionHandlerNamespace": "App/Exceptions/Handler",
   "aliases": {
@@ -23,13 +24,15 @@
         "repl",
         "web"
       ]
-    }
+    },
+    "./start/bull"
   ],
   "providers": [
     "./providers/AppProvider",
     "@adonisjs/core",
     "@adonisjs/lucid",
-    "@adonisjs/redis"
+    "@adonisjs/redis",
+    "@rocketseat/adonis-bull"
   ],
   "aceProviders": [
     "@adonisjs/repl"

server/.env.example

@@ -17,3 +17,7 @@ REDIS_PORT=6379
 REDIS_PASSWORD=
 
 RIOT_API_KEY=
+
+BULL_REDIS_HOST=127.0.0.1
+BULL_REDIS_PORT=6379
+BULL_REDIS_PASSWORD=

server/ace-manifest.json

@@ -288,6 +288,59 @@
           "alias": "c"
         }
       ]
+    },
+    "make:job": {
+      "settings": {},
+      "commandPath": "@rocketseat/adonis-bull/build/commands/MakeJob",
+      "commandName": "make:job",
+      "description": "Make a new Bull job",
+      "args": [
+        {
+          "type": "string",
+          "propertyName": "name",
+          "name": "name",
+          "required": true,
+          "description": "Name of the job class"
+        }
+      ],
+      "aliases": [],
+      "flags": []
+    },
+    "bull:exception": {
+      "settings": {},
+      "commandPath": "@rocketseat/adonis-bull/build/commands/MakeException",
+      "commandName": "bull:exception",
+      "description": "Make a Bull exception handle file",
+      "args": [],
+      "aliases": [],
+      "flags": []
+    },
+    "bull:listen": {
+      "settings": {
+        "loadApp": true,
+        "stayAlive": true
+      },
+      "commandPath": "@rocketseat/adonis-bull/build/commands/Listen",
+      "commandName": "bull:listen",
+      "description": "Start the Bull listener",
+      "args": [],
+      "aliases": [],
+      "flags": [
+        {
+          "name": "board",
+          "propertyName": "board",
+          "type": "boolean",
+          "description": "Run bull's dashboard",
+          "alias": "b"
+        },
+        {
+          "name": "port",
+          "propertyName": "port",
+          "type": "number",
+          "description": "Run bull's dashboard in the provided port",
+          "alias": "p"
+        }
+      ]
     }
   },
   "aliases": {}

server/app/Controllers/Http/SummonersController.ts

@@ -1,11 +1,13 @@
 import { HttpContextContract } from '@ioc:Adonis/Core/HttpContext'
+import Bull from '@ioc:Rocketseat/Bull'
 import { getCurrentSeason } from 'App/helpers'
+import FetchMatchList from 'App/Jobs/FetchMatchList'
 import Summoner from 'App/Models/Summoner'
 import MatchRepository from 'App/Repositories/MatchRepository'
 import BasicMatchSerializer from 'App/Serializers/BasicMatchSerializer'
 import LiveMatchSerializer from 'App/Serializers/LiveMatchSerializer'
 import Jax from 'App/Services/Jax'
-import MatchService from 'App/Services/MatchService'
+import MatchService, { MatchListMode } from 'App/Services/MatchService'
 import StatsService from 'App/Services/StatsService'
 import SummonerService from 'App/Services/SummonerService'
 import SummonerBasicValidator from 'App/Validators/SummonerBasicValidator'
@@ -47,8 +49,12 @@ export default class SummonersController {
       // Summoner names
       finalJSON.account.names = await SummonerService.getAllSummonerNames(account, summonerDB)
 
-      // MATCH LIST
+      // Only last 100 matchIds in matchlist
-      await MatchService.updateMatchList(account, region, summonerDB)
+      await MatchService.updateMatchList(account.puuid, region, MatchListMode.LIGHT)
+
+      // Add job in 1sec to load entire matchlist in DB (in background)
+      const matchListMode = summonerDB.$isLocal ? MatchListMode.FIRSTIME : MatchListMode.UPDATE
+      Bull.schedule(new FetchMatchList().key, { puuid: account.puuid, region, matchListMode }, 1000)
 
       // All seasons the summoner has played
       finalJSON.seasons = await this.getSeasons(account.puuid)
@@ -58,9 +64,8 @@ export default class SummonersController {
 
       // CURRENT GAME
       console.time('playing')
-      const currentGame = await Jax.Spectator.summonerID(account.id, region)
+      finalJSON.current = await Jax.Spectator.summonerID(account.id, region)
-      finalJSON.playing = !!currentGame
+      finalJSON.playing = !!finalJSON.current
-      finalJSON.current = currentGame
       console.timeEnd('playing')
 
       // RANKED STATS

server/app/Jobs/FetchMatchList.ts

@@ -0,0 +1,31 @@
+import { JobContract } from '@ioc:Rocketseat/Bull'
+import MatchService, { MatchListMode } from 'App/Services/MatchService'
+
+/*
+|--------------------------------------------------------------------------
+| Job setup
+|--------------------------------------------------------------------------
+|
+| This is the basic setup for creating a job, but you can override
+| some settings.
+|
+| You can get more details by looking at the bullmq documentation.
+| https://docs.bullmq.io/
+*/
+
+export interface FetchMatchListArgs {
+  puuid: string
+  region: string
+  mode: MatchListMode
+}
+
+export default class FetchMatchList implements JobContract {
+  public key = 'FetchMatchList'
+
+  public async handle(job) {
+    const { data }: { data: FetchMatchListArgs } = job
+
+    // Load entire matchlist in DB if it's first time or update it
+    await MatchService.updateMatchList(data.puuid, data.region, data.mode)
+  }
+}

server/app/Services/MatchService.ts

@@ -1,28 +1,33 @@
 import Jax from './Jax'
 import { MatchlistDto } from './Jax/src/Endpoints/MatchlistEndpoint'
-import { SummonerDTO } from './Jax/src/Endpoints/SummonerEndpoint'
-import Summoner from 'App/Models/Summoner'
 import Database from '@ioc:Adonis/Lucid/Database'
 import MatchParser from 'App/Parsers/MatchParser'
 import BasicMatchSerializer from 'App/Serializers/BasicMatchSerializer'
 import { SerializedMatch } from 'App/Serializers/SerializedTypes'
 import Match from 'App/Models/Match'
 import { notEmpty, tutorialQueues } from 'App/helpers'
+import SummonerMatchlist from 'App/Models/SummonerMatchlist'
+
+export enum MatchListMode {
+  FIRSTIME = 'firstTime',
+  UPDATE = 'update',
+  LIGHT = 'light',
+}
 
 class MatchService {
   /**
    * Add 100 matches at a time to MatchList until the stopFetching condition is true
-   * @param account of the summoner
+   * @param puuid of the summoner
    * @param region of the summoner
    * @param stopFetching condition to stop fetching the MatchList
    */
-  private async _fetchMatchListUntil(account: SummonerDTO, region: string, stopFetching: any) {
+  private async _fetchMatchListUntil(puuid: string, region: string, stopFetching: any) {
     let matchList: MatchlistDto = []
     let alreadyIn = false
     let index = 0
     do {
       console.log('--> CALL TO RIOT MATCHLIST')
-      const newMatchList = await Jax.Matchlist.puuid(account.puuid, region, index)
+      const newMatchList = await Jax.Matchlist.puuid(puuid, region, index)
       // Error while fetching Riot API
       if (!newMatchList) {
         return matchList
@@ -37,27 +42,37 @@ class MatchService {
     } while (!alreadyIn)
     return matchList
   }
+
   /**
-   * Update the full MatchList of the summoner
+   * Update the MatchList of the summoner
    */
   public async updateMatchList(
-    account: SummonerDTO,
+    puuid: string,
     region: string,
-    summonerDB: Summoner
+    fetchMode: MatchListMode
   ): Promise<MatchlistDto> {
     console.time('matchList')
 
-    const currentMatchList = await summonerDB.related('matchList').query().orderBy('matchId', 'asc')
+    const currentMatchList = await SummonerMatchlist.query()
+      .where('summoner_puuid', puuid)
+      .orderBy('matchId', 'asc')
     const currentMatchListIds = currentMatchList.map((m) => m.matchId)
 
-    console.time('RiotMatchList')
+    // Condition to stop fetching the matchlist
-    const newMatchList = await this._fetchMatchListUntil(
+    function stopFetching(newMatchList: MatchlistDto) {
-      account,
+      switch (fetchMode) {
-      region,
+        case MatchListMode.FIRSTIME:
-      (newMatchList: MatchlistDto) => {
+          return false
-        return currentMatchListIds.some((id) => id === newMatchList[newMatchList.length - 1])
+        case MatchListMode.UPDATE:
+          return currentMatchListIds.some((id) => id === newMatchList[newMatchList.length - 1])
+        case MatchListMode.LIGHT:
+        default:
+          return true
       }
-    )
+    }
+
+    console.time('RiotMatchList')
+    const newMatchList = await this._fetchMatchListUntil(puuid, region, stopFetching)
     console.timeEnd('RiotMatchList')
 
     const matchListToSave: MatchlistDto = []
@@ -73,7 +88,7 @@ class MatchService {
       await Database.table('summoner_matchlist').multiInsert(
         matchListToSave.map((id) => ({
           match_id: id,
-          summoner_puuid: summonerDB.puuid,
+          summoner_puuid: puuid,
         }))
       )
     }

server/config/bull.ts

@@ -0,0 +1,30 @@
+import Env from '@ioc:Adonis/Core/Env'
+import { BullConfig } from '@ioc:Rocketseat/Bull'
+
+/*
+|--------------------------------------------------------------------------
+| Bull configuration
+|--------------------------------------------------------------------------
+|
+| Following is the configuration used by the Bull provider to connect to
+| the redis server.
+|
+| Do make sure to pre-define the connections type inside `contracts/bull.ts`
+| file for Rocketseat/Bull to recognize connections.
+|
+| Make sure to check `contracts/bull.ts` file for defining extra connections
+*/
+
+const bullConfig: BullConfig = {
+  connection: Env.get('BULL_CONNECTION', 'local'),
+
+  connections: {
+    local: {
+      host: Env.get('BULL_REDIS_HOST', 'localhost'),
+      port: Env.get('BULL_REDIS_PORT'),
+      password: Env.get('BULL_REDIS_PASSWORD'),
+    },
+  },
+}
+
+export default bullConfig

server/contracts/bull.ts

@@ -0,0 +1,5 @@
+declare module '@ioc:Rocketseat/Bull' {
+  interface BullConnectionsList {
+    local: BullConnectionContract
+  }
+}

server/env.ts

@@ -35,4 +35,8 @@ export default Env.rules({
   REDIS_PASSWORD: Env.schema.string.optional(),
 
   RIOT_API_KEY: Env.schema.string(),
+
+  BULL_REDIS_HOST: Env.schema.string({ format: 'host' }),
+  BULL_REDIS_PORT: Env.schema.number(),
+  BULL_REDIS_PASSWORD: Env.schema.string.optional(),
 })

server/package.json

@@ -28,6 +28,7 @@
     "@adonisjs/lucid": "^16.0.2",
     "@adonisjs/redis": "^7.0.9",
     "@adonisjs/repl": "^3.1.6",
+    "@rocketseat/adonis-bull": "^1.0.4",
     "got": "^11.8.2",
     "luxon": "^2.0.2",
     "pg": "^8.7.1",

server/start/bull.ts

@@ -0,0 +1,21 @@
+/*
+|--------------------------------------------------------------------------
+| Preloaded File
+|--------------------------------------------------------------------------
+|
+| Any code written inside this file will be executed during the application
+| boot.
+|
+*/
+
+import Bull from '@ioc:Rocketseat/Bull'
+import Env from '@ioc:Adonis/Core/Env'
+
+const PORT = 9999
+const isDevelopment = Env.get('NODE_ENV') === 'development'
+
+Bull.process()
+
+if (isDevelopment) {
+  Bull.ui(PORT)
+}

server/start/jobs.ts

@@ -0,0 +1,3 @@
+const jobs = ['App/Jobs/FetchMatchList']
+
+export default jobs

server/tsconfig.json

@@ -29,7 +29,8 @@
       "@adonisjs/core",
       "@adonisjs/repl",
       "@adonisjs/lucid",
-      "@adonisjs/redis"
+      "@adonisjs/redis",
+      "@rocketseat/adonis-bull"
     ]
   }
 }