Exclude repos in the exluded list, and any repos starting with slides…

…, or course (#20)

* Exclude repos in the exluded list, starting with slides, or course

* Remove console.log statement used for testing

* Update utility function

* Make sure conda cache is actually used

* Update sleep time to 70s instead of 60s

* Filter in discussions and issues to make sure all repos have all field

.github/workflows/nextjs.yml

@@ -35,6 +35,15 @@ jobs:
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4
         with:
           node-version: '20.x'
+      - name: Get current date
+        id: date
+        run: echo "DATE=$(date +'%Y-%m')" >> $GITHUB_ENV
+      - name: Restore conda downloads cache
+        uses: actions/cache@734d9cb93d6f7610c2400b0f789eaa6f9813e271 # v3
+        with:
+          path: |
+            ~/.dashboard
+          key: conda-download-cache-${{ env.DATE }}
       - name: Collect metrics and save output
         id: metrics
         run: |
@@ -68,9 +77,6 @@ jobs:
           #
           # You may remove this line if you want to manage the configuration yourself.
           static_site_generator: next
-      - name: Get current date
-        id: date
-        run: echo "DATE=$(date +'%Y-%m')" >> $GITHUB_ENV
       - name: Restore .next cache
         uses: actions/cache@734d9cb93d6f7610c2400b0f789eaa6f9813e271 # v3
         with:
@@ -81,12 +87,6 @@ jobs:
           # If source files changed but packages didn't, rebuild from a prior cache.
           restore-keys: |
             ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json', '**/yarn.lock') }}-
-      - name: Restore conda downloads cache
-        uses: actions/cache@734d9cb93d6f7610c2400b0f789eaa6f9813e271 # v3
-        with:
-          path: |
-            ~/.dashboard
-          key: conda-download-cache-${{ env.DATE }}
       - name: Install dependencies
         run: cd "${{ github.workspace }}/app" && ${{ steps.detect-package-manager.outputs.manager }} ${{ steps.detect-package-manager.outputs.command }}
       - name: Build with Next.js

backend/excluded_repos.json

@@ -0,0 +1,17 @@
+[
+  ".github",
+  "actions",
+  "brainglobe.github.io",
+  "brainglobe-meta",
+  "HowTo",
+  "In2Research-2024",
+  "napari-experimental",
+  "neuroinformatics-unit.github.io",
+  "nuffield-students-exercises",
+  "python-cookiecutter",
+  "quarto-presentation-template",
+  "scripts",
+  "software-practice-template",
+  "software-skills",
+  "swc-slack"
+]

backend/src/fetchers/discussions.ts

@@ -3,6 +3,7 @@
 import { Organization } from '@octokit/graphql-schema';
 import { Config, Fetcher } from '..';
 import { CustomOctokit } from '../lib/octokit';
+import excludedRepos from '../../excluded_repos.json';
 
 const queryForDiscussions = async (octokit: CustomOctokit, config: Config) => {
   return await octokit.graphql.paginate<{ organization: Organization }>(
@@ -52,7 +53,14 @@ export const addDiscussionData: Fetcher = async (result, octokit, config) => {
     return result;
   }
 
-  for (const repo of dataResult) {
+  const filteredResult = dataResult.filter(
+    (repo) =>
+      !(excludedRepos.includes(repo!.repositoryName) ||
+        repo!.repositoryName.startsWith("slides-") ||
+        repo!.repositoryName.startsWith("course-")),
+  )
+
+  for (const repo of filteredResult) {
     result.repositories[repo.repositoryName].discussionsCount =
       repo.discussionsCount;
   }

backend/src/fetchers/downloads_pepy.ts

@@ -25,7 +25,7 @@ export interface PePyResult {
 const fetchDownloads = async (projectName: string) => {
   let retries = 2;
   // PePy API has a rate limit of 10 requests per minute
-  let sleep_time = 60_000;
+  let sleep_time = 70_000;
   const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
   while (retries > 0) {
     try {

backend/src/fetchers/fetcher_utils.ts

@@ -1,6 +1,7 @@
 import { CustomOctokit } from '../lib/octokit';
 import { Config } from '../index';
 import { Organization, Repository } from '@octokit/graphql-schema';
+import excludedRepos from '../../excluded_repos.json';
 
 export const queryRepoNames = async (octokit: CustomOctokit, config: Config) => {
   const organization = await octokit.graphql.paginate<{
@@ -29,7 +30,10 @@ export const queryRepoNames = async (octokit: CustomOctokit, config: Config) =>
 
   return organization.organization.repositories.nodes!.filter(
     (repo) =>
-      !(repo?.isArchived && !config.includeArchived) ||
-      !(repo.isFork && !config.includeForks),
+      (!(repo?.isArchived && !config.includeArchived) ||
+        !(repo.isFork && !config.includeForks)) &&
+      !(excludedRepos.includes(repo!.name) ||
+        repo!.name.startsWith("slides-") ||
+        repo!.name.startsWith("course-")),
   ) as Repository[];
 };

backend/src/fetchers/issues.ts

@@ -9,6 +9,7 @@ import {
 } from '@octokit/graphql-schema';
 import { Config, Fetcher } from '..';
 import { CustomOctokit } from '../lib/octokit';
+import excludedRepos from '../../excluded_repos.json';
 
 const getIssueAndPrData = async (octokit: CustomOctokit, config: Config) => {
   const issueData = await octokit.graphql.paginate<{
@@ -76,7 +77,12 @@ const getIssueAndPrData = async (octokit: CustomOctokit, config: Config) => {
 
 export const addIssueAndPrData: Fetcher = async (result, octokit, config) => {
   const dataResult = await getIssueAndPrData(octokit, config);
-  dataResult.organization.repositories.nodes.forEach((repo) => {
+  dataResult.organization.repositories.nodes.filter(
+    (repo) =>
+      !(excludedRepos.includes(repo!.name) ||
+        repo!.name.startsWith("slides-") ||
+        repo!.name.startsWith("course-")),
+  ).forEach((repo) => {
     result.repositories[repo.name] = {
       ...result.repositories[repo.name],
       totalIssuesCount: repo.totalIssues.totalCount,

backend/src/fetchers/repository.ts

@@ -3,6 +3,7 @@
 import { Organization, Repository } from '@octokit/graphql-schema';
 import { Fetcher } from '..';
 import { RepositoryResult } from '../../../types';
+import excludedRepos from '../../excluded_repos.json';
 
 export const addRepositoriesToResult: Fetcher = async (
   result,
@@ -65,8 +66,11 @@ export const addRepositoriesToResult: Fetcher = async (
 
   const filteredRepos = organization.organization.repositories.nodes!.filter(
     (repo) =>
-      !(repo?.isArchived && !config.includeArchived) ||
-      !(repo.isFork && !config.includeForks),
+      (!(repo?.isArchived && !config.includeArchived) ||
+      !(repo.isFork && !config.includeForks)) &&
+      !(excludedRepos.includes(repo!.name) ||
+      repo!.name.startsWith("slides-") ||
+      repo!.name.startsWith("course-")),
   ) as Repository[];
 
   // Just in case the filteredRepos is not stably ordered