UPDATE: psl and tld improvements

src/routers/changelogRouter.ts → src/ChangeLogUI.ts

@@ -1,19 +1,18 @@
-import path from 'path';
+import Handlebars from 'handlebars';
 import Router from 'koa-router';
-
-import { ChangeLog } from '../changelog';
 import { DateTime } from 'luxon';
 
+import { ChangeLog } from './changelog';
 
 class ChangeLogUI {
     changelogRouter: Router;
 
     constructor(
-        private changeLog:ChangeLog,
-        private home:string,
-        private mount:string,
-        private title:string,
-        private aplink:string,
+        private changeLog: ChangeLog,
+        private home: string,
+        private mount: string,
+        private title: string,
+        private aplink: string,
     ) {
         this.init();
     }
@@ -31,6 +30,7 @@ class ChangeLogUI {
                 aplink: this.aplink,
                 count: this.changeLog.getKeys().length,
                 data: this.changeLog.getAll(),
+                h1: new Handlebars.SafeString(`<a href="../index.html">${Handlebars.escapeExpression(this.title)}</a> Change Log`),
                 rssUrl: `https://resolve.rs${this.mount}/rss.xml`,
                 title: `${this.title} Change Log`,
             });
@@ -43,6 +43,8 @@ class ChangeLogUI {
             if (data) {
                 ctx.body = await ctx.render('_changelog/_index.hbs', {
                     data,
+                    date,
+                    h1: new Handlebars.SafeString(`<a href="../index.html">${Handlebars.escapeExpression(this.title)}</a> Changes on ${date}`),
                     next: this.changeLog.getNext(date),
                     previous: this.changeLog.getPrevious(date),
                     title: `${this.title} Changes on ${date}`,
@@ -56,7 +58,7 @@ class ChangeLogUI {
 
             const startDate = this.changeLog.getFirst().date;
             const pubDate = DateTime.fromISO(startDate, { zone: 'utc' }).toRFC2822();
-            
+
             ctx.body = await ctx.render('_changelog/rss.hbs', {
                 data: Object.values(this.changeLog.getAll()).slice(0, 20),
                 description: `Monitor changes to the ${this.title} via RSS`,
@@ -98,29 +100,4 @@ class ChangeLogUI {
     }
 }
 
-const pslUI:ChangeLogUI = new ChangeLogUI(
-    new ChangeLog(path.join(__dirname, '../../data/publicsuffix/deltas')),
-    '/psl/index.html',
-    '/psl/changelog',
-    'Public Suffix List',
-    'https://botsin.space/@PublicSuffixChanges',
-);
-const pslChangelogRouter = pslUI.changelogRouter;
-const pslGetUrls = pslUI.getUrls;
-
-const tldUI: ChangeLogUI = new ChangeLogUI(
-    new ChangeLog(path.join(__dirname, '../../data/icann/deltas')),
-    '/tlds/index.html',
-    '/tlds/changelog',
-    'ICANN TLD',
-    'https://botsin.space/@TLDChanges',
-);
-const tldChangelogRouter = tldUI.changelogRouter;
-const tldGetUrls = tldUI.getUrls;
-
-export {
-    pslChangelogRouter,
-    pslGetUrls,
-    tldChangelogRouter,
-    tldGetUrls,
-}
+export { ChangeLogUI }

src/actions/sitemap.ts

@@ -6,7 +6,7 @@ import * as infoRouter from '../routers/infoRouter';
 import * as ipRouter from '../routers/ipRouter';
 import * as tldsRouter from '../routers/tldsRouter';
 import * as resolverRouter from '../routers/resolverRouter';
-import { pslGetUrls, tldGetUrls } from '../routers/changelogRouter';
+import { pslChangeLogGetUrls } from '../routers/pslRouter';
 
 async function sitemap(ctx:any) {
 
@@ -19,9 +19,9 @@ async function sitemap(ctx:any) {
     urls.push(...infoRouter.getUrls());
     urls.push(...ipRouter.getUrls());
     urls.push(...resolverRouter.getUrls());
+    urls.push(...pslChangeLogGetUrls());
     urls.push(...tldsRouter.getUrls());
-    urls.push(...pslGetUrls());
-    urls.push(...tldGetUrls());
+    urls.push(...tldsRouter.tldsChangeLogGetUrls());
 
     // hard-coded to avoid circular dependencies
     urls.push("/");

src/changelog.ts

@@ -46,7 +46,7 @@ export class ChangeLog {
         } catch (err) {
             logger.error({ err, dir }, 'Error reading ChangeLog directory');
         }
-        logger.info({ count: this.cache.size, dir }, 'Files loaded into ChangeLog');
+        logger.info({ count: Object.keys(this.cache).length, dir }, 'Files loaded into ChangeLog');
     }
 
     getAll() {

src/data/domainData.ts

@@ -5,7 +5,7 @@ import * as punycode from 'punycode';
 
 const publicSuffixes:string[] = [];
 const icannTlds:string[] = [];
-const pslTlds:string[] = [];
+const pslTlds:{ [key: string]: string[] } = {};
 const niceTlds:string[] = [];
 const usableTlds:string[] = [];
 
@@ -27,18 +27,16 @@ async function initialize(logger:Pino.Logger) {
     publicSuffixes.filter(line => line.startsWith('*.') && line.indexOf('.', 3) == -1).forEach(line => usableTlds.push(line.slice(2)));
     usableTlds.sort();
 
-    const tldSet = new Set<string>();
     for (const domain of publicSuffixes) {
         const parts = domain.split('.');
         const last = parts[parts.length - 1].trim();
-        if (tldSet.has(last)) {
-            continue;
+        if (!pslTlds[last]) {
+            pslTlds[last] = [domain];
+        } else {
+            pslTlds[last].push(domain);
         }
-        tldSet.add(last);
     }
-
-    pslTlds.push(...tldSet.values());       //LATER: include ones that are in public suffix with a dot (i.e. that don't allow top-level registration)
-    pslTlds.sort();
+    //LATER: include ones that are in public suffix with a dot (i.e. that don't allow top-level registration)
 
     const niceFileName = path.join(__dirname, '../..', 'data', 'domain-suffix.txt');
 

src/routers/domainRouter.ts

@@ -1,7 +1,5 @@
 import KoaRouter from 'koa-router';
-import Handlebars from 'handlebars';
 import * as punycode from 'punycode';
-import * as psl from 'psl';
 import * as wsw from 'whoisserver-world'
 
 import * as domainData from '../data/domainData';
@@ -70,10 +68,10 @@ domainRouter.get('/domains/icann-vs-psl.html', async (ctx:any) => {
 
     const icannOnly = new Set<string>();
     domainData.icannTlds.forEach( domain => icannOnly.add(domain));
-    domainData.pslTlds.forEach( domain => icannOnly.delete(domain));
+    Object.keys(domainData.pslTlds).forEach( domain => icannOnly.delete(domain));
 
     const pslOnly = new Set<string>();
-    domainData.pslTlds.forEach( domain => pslOnly.add(domain));
+    Object.keys(domainData.pslTlds).forEach( domain => pslOnly.add(domain));
     domainData.icannTlds.forEach( domain => pslOnly.delete(domain));
 
     ctx.body = await ctx.render('domains/icann-vs-psl.hbs', {
@@ -102,14 +100,6 @@ domainRouter.get('/domains/icann-vs-wsw.html', async (ctx:any) => {
      });
 });
 
-domainRouter.get('/domains/psl-tlds.html', async (ctx:any) => {
-    ctx.redirect(ctx, '/psl/index.html');
-});
-
-domainRouter.get('/psl/', async (ctx: any) => {
-    ctx.redirect(ctx, '/psl/index.html');
-});
-
 domainRouter.get('/domains/publicsuffix.html', async (ctx: any) => {
     ctx.redirect(ctx, '/psl/test.html');
 });
@@ -118,40 +108,8 @@ domainRouter.post('/domains/publicsuffix.html', async (ctx: any) => {
     ctx.redirect(ctx, `/psl/test.html?q=${encodeURIComponent(ctx.request.body.hostname)}`);
 });
 
-domainRouter.get('/psl/index.html', async (ctx: any) => {
-    ctx.body = await ctx.render('psl/index.hbs', {
-        domains: domainData.pslTlds,
-        title: 'Public Suffix List Top Level Domains',
-     });
-});
-
-domainRouter.get('/psl/test.html', async (ctx:any) => {
-
-    const q = ctx.request.query.q;
-    const hostname = q;
-    let get:string|null|undefined;
-    let parsed:psl.ParsedDomain|psl.ParseError|null|undefined;
-    if (hostname) {
-        if (!psl.isValid(hostname)) {
-            ctx.flash('error', `${Handlebars.escapeExpression(hostname)} is not a valid public domain!`);
-        } else {
-            get = psl.get(hostname);
-            parsed = psl.parse(hostname);
-
-            if (parsed.error) {
-                ctx.flash('error', `${parsed.error.message} for ${Handlebars.escapeExpression(hostname)}`);
-                parsed = null;
-            }   
-        }
-    }
-
-    ctx.body = await ctx.render('psl/test.hbs', {
-        examples: ['test.github.io', 'github.io'],
-        get,
-        parsed,
-        q,
-        title: 'Public Suffix Test',
-     });
+domainRouter.get('/domains/psl-tlds.html', async (ctx:any) => {
+    ctx.redirect(ctx, '/psl/index.html');
 });
 
 domainRouter.get('/domains/usable-tlds.html', async (ctx:any) => {

src/routers/pslRouter.ts

@@ -0,0 +1,70 @@
+import Handlebars from 'handlebars';
+import KoaRouter from 'koa-router';
+import path from 'path';
+import * as psl from 'psl';
+
+import { ChangeLog } from '../changelog';
+import { ChangeLogUI } from '../ChangeLogUI';
+import * as domainData from '../data/domainData';
+
+const pslRouter = new KoaRouter();
+
+pslRouter.get('/psl/', async (ctx: any) => {
+    ctx.redirect(ctx, '/psl/index.html');
+});
+
+
+pslRouter.get('/psl/index.html', async (ctx: any) => {
+
+    ctx.body = await ctx.render('psl/index.hbs', {
+        domains: Object.keys(domainData.pslTlds).sort((a, b) => { return a.localeCompare(b); }),
+        psl: domainData.pslTlds,
+        title: 'Public Suffix List Top Level Domains',
+        total: Object.values(domainData.pslTlds).reduce((acc, val) => acc + val.length, 0),
+    });
+});
+
+pslRouter.get('/psl/test.html', async (ctx: any) => {
+
+    const q = ctx.request.query.q;
+    const hostname = q;
+    let get: string | null | undefined;
+    let parsed: psl.ParsedDomain | psl.ParseError | null | undefined;
+    if (hostname) {
+        if (!psl.isValid(hostname)) {
+            ctx.flash('error', `${Handlebars.escapeExpression(hostname)} is not a valid public domain!`);
+        } else {
+            get = psl.get(hostname);
+            parsed = psl.parse(hostname);
+
+            if (parsed.error) {
+                ctx.flash('error', `${parsed.error.message} for ${Handlebars.escapeExpression(hostname)}`);
+                parsed = null;
+            }
+        }
+    }
+
+    ctx.body = await ctx.render('psl/test.hbs', {
+        examples: ['test.github.io', 'github.io'],
+        get,
+        parsed,
+        q,
+        title: 'Public Suffix Test',
+    });
+});
+
+const pslChangeLogUI:ChangeLogUI = new ChangeLogUI(
+    new ChangeLog(path.join(__dirname, '../../data/publicsuffix/deltas')),
+    '/psl/index.html',
+    '/psl/changelog',
+    'Public Suffix List',
+    'https://botsin.space/@PublicSuffixChanges',
+);
+const pslChangeLogRouter = pslChangeLogUI.changelogRouter;
+const pslChangeLogGetUrls = pslChangeLogUI.getUrls;
+
+export {
+    pslRouter,
+    pslChangeLogRouter,
+    pslChangeLogGetUrls,
+}

src/routers/tldsRouter.ts

@@ -1,8 +1,12 @@
 import * as punycode from 'punycode';
 import Router from 'koa-router';
+import path from 'path';
 import * as wsw from 'whoisserver-world'
 
 import * as rdapData from '../data/rdapData';
+import { ChangeLog } from '../changelog';
+import { ChangeLogUI } from '../ChangeLogUI';
+import * as domainData from '../data/domainData';
 
 const tldsRouter = new Router();
 
@@ -72,6 +76,7 @@ tldsRouter.get('/tlds/:tld/index.html', async (ctx:any) => {
     tldInfo.unicode = punycode.toUnicode(tld);
 
     ctx.body = await ctx.render('_tlds/index.hbs', {
+        publicSuffixes: domainData.pslTlds[tld],
         rdapUnofficial,
         sampleDomains,
         title: `Top Level Domain "${tld}"`,
@@ -92,7 +97,19 @@ function getUrls():string[] {
     return retVal;
 }
 
+const tldChangeLogUI: ChangeLogUI = new ChangeLogUI(
+    new ChangeLog(path.join(__dirname, '../../data/icann/deltas')),
+    '/tlds/index.html',
+    '/tlds/changelog',
+    'ICANN TLD',
+    'https://botsin.space/@TLDChanges',
+);
+const tldsChangeLogRouter = tldChangeLogUI.changelogRouter;
+const tldsChangeLogGetUrls = tldChangeLogUI.getUrls;
+
 export {
     tldsRouter,
-    getUrls
+    getUrls,
+    tldsChangeLogRouter,
+    tldsChangeLogGetUrls,
 }

src/server.ts

@@ -17,17 +17,17 @@ import * as transliteration from 'transliteration';
 import config from './config';
 import * as asn from './data/maxmindData';
 import { resolverRouter } from './routers/resolverRouter';
-import { pslChangelogRouter, tldChangelogRouter } from './routers/changelogRouter';
 import { domainRouter } from './routers/domainRouter';
 import { httpRouter } from './routers/httpRouter';
 import { ipRouter } from './routers/ipRouter';
 import { logger , options as loggerOptions } from './logger';
 import { cryptoRouter } from './routers/cryptoRouter';
 import { datagenRouter } from './routers/datagenRouter';
 import { dnsRouter } from './routers/dnsRouter';
+import { pslRouter, pslChangeLogRouter } from './routers/pslRouter';
 import * as resolvers from './data/resolverData';
 import { rootRouter } from './routers/rootRouter';
-import { tldsRouter } from './routers/tldsRouter';
+import { tldsRouter, tldsChangeLogRouter } from './routers/tldsRouter';
 import { infoRouter } from './routers/infoRouter';
 import * as domains from './data/domainData';
 import * as rdapData from './data/rdapData';
@@ -124,6 +124,9 @@ app.use(KoaViews(path.join(__dirname, '..', 'views'), {
                 return result;
             },
             fromPunycode: function(domain:string) { return domain ? punycode.toUnicode(domain) : '(null)'; },
+            get: function (map: { [key: string]: any}, key:string) {
+                return map[key];
+            },
             ifDomainUnicode: function(context:any, options:any) { return context && !context.match(/^[a-z]+$/) ? options.fn(this) : options.inverse(this); },
             integerFormat: function(theNumber:any):string {
                 if (theNumber == 0) {
@@ -140,6 +143,9 @@ app.use(KoaViews(path.join(__dirname, '..', 'views'), {
             isArray: function(target:any) {
                 return target && Array.isArray(target);
             },
+            length: function(target:any) {
+                return target ? target.length : 0;
+            },
             noCacheDev: function(multiParam:boolean) {
               if (process.env.COMMIT) {
                 return '';
@@ -221,8 +227,9 @@ app.use(domainRouter.routes());
 app.use(httpRouter.routes());
 app.use(ipRouter.routes());
 app.use(infoRouter.routes());
-app.use(pslChangelogRouter.routes());
-app.use(tldChangelogRouter.routes());
+app.use(pslChangeLogRouter.routes());
+app.use(pslRouter.routes());
+app.use(tldsChangeLogRouter.routes());
 app.use(tldsRouter.routes());
 
 async function main() {