Dolt

.gitignore

@@ -213,3 +213,10 @@ curriculum/build
 ### Playwright ###
 
 /playwright
+
+### Dolt ###
+.sqlhistory
+.dolt*
+dolt/api/curriculum.json
+dolt/api/learn-curriculum.json
+dolt/api/mobile-curriculum.json

.prettierignore

@@ -18,3 +18,4 @@ shared/utils/get-lines.test.js
 shared/utils/is-audited.js
 shared/utils/validate.js
 shared/utils/validate.test.js
+tools/challenge-parser/parser/super-challenge.md

client/utils/build-challenges.js

@@ -1,5 +1,5 @@
 const path = require('path');
-
+const { writeFileSync } = require('fs');
 const _ = require('lodash');
 
 const envData = require('../config/env.json');
@@ -38,6 +38,10 @@ exports.replaceChallengeNode = () => {
 
 exports.buildChallenges = async function buildChallenges() {
   const curriculum = await getChallengesForLang(curriculumLocale);
+  writeFileSync(
+    `../dolt/api/learn-curriculum.json`,
+    JSON.stringify(curriculum, null, 2)
+  );
   const superBlocks = Object.keys(curriculum);
   const blocks = superBlocks
     .map(superBlock => curriculum[superBlock].blocks)

curriculum/capi-connector.js

@@ -0,0 +1,189 @@
+const { readFile, writeFile } = require('fs/promises');
+
+async function fetchCapCurriculum() {
+  try {
+    const response = await fetch('http://localhost:3010/curriculum');
+    const data = await response.json();
+    return data;
+  } catch (error) {
+    console.error('Error fetching curriculum data:', error);
+    throw error;
+  }
+}
+
+function getChallengeFromPath(capCurriculum, englishPath) {
+  return capCurriculum.find(
+    c =>
+      englishPath.endsWith(`${c.block}/${c.objectId}.md`) ||
+      englishPath.endsWith(`${c.block}/${c.challengeDashedName}.md`)
+  );
+}
+
+function getCapDescription(capCurriculum, englishPath) {
+  const challenge = getChallengeFromPath(capCurriculum, englishPath);
+
+  return challenge?.description
+    ? `<section id="description">${challenge.description}</section>`
+    : '';
+}
+
+function getCapInstructions(capCurriculum, englishPath) {
+  const challenge = getChallengeFromPath(capCurriculum, englishPath);
+
+  return challenge?.instructions
+    ? `<section id="instruction">${challenge.instructions}</section>`
+    : '';
+}
+
+/**
+ * Compares the challenge in the Curriculum Database with the challenge in the file system after parsing.
+ *
+ * Stores a log of the differences at `/curriculum/cdb-progress.json`
+ *
+ * ```json
+ * [{
+ *   "challenge_id": string,
+ *   "property": string,
+ *   "expected": JSON,
+ *   "actual": JSON | undefined
+ * }]
+ * ```
+ * @param {Object} file - The file information
+ * @param {Record<string, unknown>} file.challenge - The challenge information
+ * @param {string} file.lang - The langauge of the file
+ */
+async function compareCDBToFS({ challenge, lang }) {
+  const challenges = await getChallenges();
+  const matching_challenge = challenges.find(({ id }) => id === challenge.id);
+
+  if (!matching_challenge) {
+    await addToLog('./cdb-progress.json', {
+      challenge_id: challenge.id,
+      property: null,
+      expected: challenge,
+      actual: undefined
+    });
+  }
+
+  const challenge_keys = Object.keys(challenge);
+
+  for (const key of challenge_keys) {
+    if (!isDeepEqual(challenge[key], matching_challenge[key])) {
+      await addToLog({
+        challenge_id: challenge.id,
+        property: key,
+        expected: challenge[key],
+        actual: matching_challenge[key]
+      });
+    }
+  }
+}
+
+function isDeepEqual(a, b) {
+  if (a === b) {
+    return true;
+  }
+
+  if (typeof a !== typeof b) {
+    return false;
+  }
+
+  if (typeof a === 'object' && a !== null && b !== null) {
+    const a_keys = Object.keys(a);
+    const b_keys = Object.keys(b);
+
+    if (a_keys.length !== b_keys.length) {
+      return false;
+    }
+
+    for (const key of a_keys) {
+      if (!isDeepEqual(a[key], b[key])) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  return false;
+}
+
+async function addToLog(value) {
+  const data = await readFile('./cdb-progress.json');
+  const logs = JSON.parse(data);
+
+  if (logs.length === 0) {
+    await writeFile('./cdb-progress.json', JSON.stringify([value]));
+  } else {
+    logs.push(value);
+    await writeFile('./cdb-progress.json', JSON.stringify(logs));
+  }
+}
+
+/**
+ * @type {{
+ *     audioPath?: string,
+ *     block: string,
+ *     blockId?: string,
+ *     challengeOrder?: number,
+ *     certification: string,
+ *     challengeType: number,
+ *     dashedName: string,
+ *     description?: string,
+ *     disableLoopProtectTests: boolean,
+ *     disableLoopProtectPreview: boolean,
+ *     challengeFiles?: { fileKey: string, ext: string, name: string, editableRegionBoundaries: number[], path: string, head: string, tail: string, seed: string, contents: string, id: string, history: string[]},
+ *     guideUrl?: string,
+ *     hasEditableBoundaries?: boolean,
+ *     helpCategory?: string,
+ *     videoUrl?: string,
+ *     fillInTheBlank?: { sentence: string, blanks: { answer: string, feedback: string }[] },
+ *     forumTopicId?: number,
+ *     id: string,
+ *     instructions?: string,
+ *     isComingSoon?: boolean,
+ *     isLocked?: boolean,
+ *     isPrivate?: boolean,
+ *     msTrophyId?: string,
+ *     notes?: string,
+ *     order?: number,
+ *     prerequisites?: { id: string, title: string }[],
+ *     videoId?: string,
+ *     videoLocaleIds?: string[],
+ *     bilibiliIds?: { aid: number, bvid: string, cid: number },
+ *     question?: { text: string, answers: { answer: string, feedback: string }[] },
+ *     required?: { link: string, raw: boolean, src: string, crossDomain: boolean}[],
+ *     assignments?: string[],
+ *     scene?: { setup: { code: string, language: string }, commands: { command: string, code: string, language: string }[] },
+ *     solutions?: { [T in FileKey]: FileKeyChallenge },
+ *     superBlock: string,
+ *     superOrder?: number,
+ *     tests: { id: string, text: string, testString?: string, title?: string }[],
+ *     template?: string,
+ *     title: string,
+ *     time?: string,
+ *     translationPending: boolean,
+ *     url?: string,
+ *     usesMultifileEditor?: boolean,
+ *   }}
+ */
+let curriculum;
+
+async function getChallenges(lang = 'english') {
+  if (curriculum) {
+    return curriculum;
+  }
+
+  curriculum = await fetch(
+    `http://localhost:3010/curriculum?lang=${lang}`
+  ).then(res => res.json());
+
+  return curriculum;
+}
+
+module.exports = {
+  compareCDBToFS,
+  getCapDescription,
+  getCapInstructions,
+  fetchCapCurriculum
+};

curriculum/get-challenges.js

@@ -19,6 +19,8 @@ const { createPoly } = require('../shared/utils/polyvinyl');
 const { getSuperOrder, getSuperBlockFromDir } = require('./utils');
 const { metaSchemaValidator } = require('./schema/meta-schema');
 
+// const { compareCDBToFS } = require('./capi-connector');
+
 const access = util.promisify(fs.access);
 
 const CHALLENGES_DIR = path.resolve(__dirname, 'challenges');
@@ -317,6 +319,8 @@ function generateChallengeCreator(lang, englishPath, i18nPath) {
     addMetaToChallenge(challenge, meta);
     fixChallengeProperties(challenge);
 
+    // await compareCDBToFS({ challenge, lang });
+
     return challenge;
   }
   return createChallenge;

dolt/README.md

@@ -0,0 +1,53 @@
+# Curriculum Database
+
+## Usage
+
+1. Start the SQL server:
+
+```bash
+dolt sql-server
+```
+
+2. Run commands/scripts
+3. Stop the SQL server
+4. Commit changes:
+
+```bash
+dolt add .
+dolt commit -m "message"
+dolt push origin main
+```
+
+**NOTE:** If you drop the database, you will have a bad time. Instead drop all the tables for a clean start. See `queries/drop-tables.sql`
+
+## Notes
+
+The database name in Dolt is the directory name. This is NOT configurable.
+
+Make a query with a file
+
+```bash
+mysql --host 127.0.0.1 --port 3306 -u root -p dolt < queries/<name>.sql
+# OR
+dolt sql < queries/<name>.sql
+```
+
+```bash
+DATABASE_URL=mysql://root:@127.0.0.1/dolt
+```
+
+MYSQL has reserved words: https://dev.mysql.com/doc/refman/8.0/en/keywords.html
+
+- `description` has been renamed to `descriptions`
+- `url` has been renamed to `course_url`
+- `required` has been renamed to `required_resources`
+
+- [x] `fields` to go
+- [x] `time` to stay, but be added per block **not** per challenge
+  - Turned into `block_time_to_complete` table
+- `disableLoopProtect*` to go?
+  - Maybe, because we can.
+- [x] `hasEditableBoundaries` to go
+  - `if challengeFiles && challengeFiles.editabledBoundaries.length > 0`
+- [x] `certification.slug` keep out of database
+- [ ] `certification.dashed_name` remove

dolt/api/README.md

@@ -0,0 +1,8 @@
+Run the main repo,
+Two JSON files will be created in dolt/api/
+
+Clone the DB and run the server
+Start the API with `node index.js` while in the dolt folder
+It will write a third JSON file in dolt/api/
+
+Run `npx jest curriculum.test.js` while in the dolt folder to test the JSON created from the API against one of the ones created from the client