From 0529f301db835516e996589e770504d3e071ff5b Mon Sep 17 00:00:00 2001 From: Andrew Marcuse Date: Sat, 3 Feb 2024 00:08:30 -0500 Subject: [PATCH] UPDATE: psl and tld improvements --- .../changelogRouter.ts => ChangeLogUI.ts} | 47 ++++--------- src/actions/sitemap.ts | 6 +- src/changelog.ts | 2 +- src/data/domainData.ts | 14 ++-- src/routers/domainRouter.ts | 50 ++----------- src/routers/pslRouter.ts | 70 +++++++++++++++++++ src/routers/tldsRouter.ts | 19 ++++- src/server.ts | 15 ++-- views/_tlds/index.hbs | 13 +++- views/psl/index.hbs | 29 ++++++-- views/tlds/index.hbs | 2 +- 11 files changed, 160 insertions(+), 107 deletions(-) rename src/{routers/changelogRouter.ts => ChangeLogUI.ts} (74%) create mode 100644 src/routers/pslRouter.ts diff --git a/src/routers/changelogRouter.ts b/src/ChangeLogUI.ts similarity index 74% rename from src/routers/changelogRouter.ts rename to src/ChangeLogUI.ts index 3e92d67..4c4396f 100644 --- a/src/routers/changelogRouter.ts +++ b/src/ChangeLogUI.ts @@ -1,19 +1,18 @@ -import path from 'path'; +import Handlebars from 'handlebars'; import Router from 'koa-router'; - -import { ChangeLog } from '../changelog'; import { DateTime } from 'luxon'; +import { ChangeLog } from './changelog'; class ChangeLogUI { changelogRouter: Router; constructor( - private changeLog:ChangeLog, - private home:string, - private mount:string, - private title:string, - private aplink:string, + private changeLog: ChangeLog, + private home: string, + private mount: string, + private title: string, + private aplink: string, ) { this.init(); } @@ -31,6 +30,7 @@ class ChangeLogUI { aplink: this.aplink, count: this.changeLog.getKeys().length, data: this.changeLog.getAll(), + h1: new Handlebars.SafeString(`${Handlebars.escapeExpression(this.title)} Change Log`), rssUrl: `https://resolve.rs${this.mount}/rss.xml`, title: `${this.title} Change Log`, }); @@ -43,6 +43,8 @@ class ChangeLogUI { if (data) { ctx.body = await ctx.render('_changelog/_index.hbs', { data, + date, + h1: new Handlebars.SafeString(`${Handlebars.escapeExpression(this.title)} Changes on ${date}`), next: this.changeLog.getNext(date), previous: this.changeLog.getPrevious(date), title: `${this.title} Changes on ${date}`, @@ -56,7 +58,7 @@ class ChangeLogUI { const startDate = this.changeLog.getFirst().date; const pubDate = DateTime.fromISO(startDate, { zone: 'utc' }).toRFC2822(); - + ctx.body = await ctx.render('_changelog/rss.hbs', { data: Object.values(this.changeLog.getAll()).slice(0, 20), description: `Monitor changes to the ${this.title} via RSS`, @@ -98,29 +100,4 @@ class ChangeLogUI { } } -const pslUI:ChangeLogUI = new ChangeLogUI( - new ChangeLog(path.join(__dirname, '../../data/publicsuffix/deltas')), - '/psl/index.html', - '/psl/changelog', - 'Public Suffix List', - 'https://botsin.space/@PublicSuffixChanges', -); -const pslChangelogRouter = pslUI.changelogRouter; -const pslGetUrls = pslUI.getUrls; - -const tldUI: ChangeLogUI = new ChangeLogUI( - new ChangeLog(path.join(__dirname, '../../data/icann/deltas')), - '/tlds/index.html', - '/tlds/changelog', - 'ICANN TLD', - 'https://botsin.space/@TLDChanges', -); -const tldChangelogRouter = tldUI.changelogRouter; -const tldGetUrls = tldUI.getUrls; - -export { - pslChangelogRouter, - pslGetUrls, - tldChangelogRouter, - tldGetUrls, -} +export { ChangeLogUI } \ No newline at end of file diff --git a/src/actions/sitemap.ts b/src/actions/sitemap.ts index 46be852..3472971 100644 --- a/src/actions/sitemap.ts +++ b/src/actions/sitemap.ts @@ -6,7 +6,7 @@ import * as infoRouter from '../routers/infoRouter'; import * as ipRouter from '../routers/ipRouter'; import * as tldsRouter from '../routers/tldsRouter'; import * as resolverRouter from '../routers/resolverRouter'; -import { pslGetUrls, tldGetUrls } from '../routers/changelogRouter'; +import { pslChangeLogGetUrls } from '../routers/pslRouter'; async function sitemap(ctx:any) { @@ -19,9 +19,9 @@ async function sitemap(ctx:any) { urls.push(...infoRouter.getUrls()); urls.push(...ipRouter.getUrls()); urls.push(...resolverRouter.getUrls()); + urls.push(...pslChangeLogGetUrls()); urls.push(...tldsRouter.getUrls()); - urls.push(...pslGetUrls()); - urls.push(...tldGetUrls()); + urls.push(...tldsRouter.tldsChangeLogGetUrls()); // hard-coded to avoid circular dependencies urls.push("/"); diff --git a/src/changelog.ts b/src/changelog.ts index fb290eb..cc81577 100644 --- a/src/changelog.ts +++ b/src/changelog.ts @@ -46,7 +46,7 @@ export class ChangeLog { } catch (err) { logger.error({ err, dir }, 'Error reading ChangeLog directory'); } - logger.info({ count: this.cache.size, dir }, 'Files loaded into ChangeLog'); + logger.info({ count: Object.keys(this.cache).length, dir }, 'Files loaded into ChangeLog'); } getAll() { diff --git a/src/data/domainData.ts b/src/data/domainData.ts index d790a0a..5f9f61c 100644 --- a/src/data/domainData.ts +++ b/src/data/domainData.ts @@ -5,7 +5,7 @@ import * as punycode from 'punycode'; const publicSuffixes:string[] = []; const icannTlds:string[] = []; -const pslTlds:string[] = []; +const pslTlds:{ [key: string]: string[] } = {}; const niceTlds:string[] = []; const usableTlds:string[] = []; @@ -27,18 +27,16 @@ async function initialize(logger:Pino.Logger) { publicSuffixes.filter(line => line.startsWith('*.') && line.indexOf('.', 3) == -1).forEach(line => usableTlds.push(line.slice(2))); usableTlds.sort(); - const tldSet = new Set(); for (const domain of publicSuffixes) { const parts = domain.split('.'); const last = parts[parts.length - 1].trim(); - if (tldSet.has(last)) { - continue; + if (!pslTlds[last]) { + pslTlds[last] = [domain]; + } else { + pslTlds[last].push(domain); } - tldSet.add(last); } - - pslTlds.push(...tldSet.values()); //LATER: include ones that are in public suffix with a dot (i.e. that don't allow top-level registration) - pslTlds.sort(); + //LATER: include ones that are in public suffix with a dot (i.e. that don't allow top-level registration) const niceFileName = path.join(__dirname, '../..', 'data', 'domain-suffix.txt'); diff --git a/src/routers/domainRouter.ts b/src/routers/domainRouter.ts index c915a04..8356a57 100644 --- a/src/routers/domainRouter.ts +++ b/src/routers/domainRouter.ts @@ -1,7 +1,5 @@ import KoaRouter from 'koa-router'; -import Handlebars from 'handlebars'; import * as punycode from 'punycode'; -import * as psl from 'psl'; import * as wsw from 'whoisserver-world' import * as domainData from '../data/domainData'; @@ -70,10 +68,10 @@ domainRouter.get('/domains/icann-vs-psl.html', async (ctx:any) => { const icannOnly = new Set(); domainData.icannTlds.forEach( domain => icannOnly.add(domain)); - domainData.pslTlds.forEach( domain => icannOnly.delete(domain)); + Object.keys(domainData.pslTlds).forEach( domain => icannOnly.delete(domain)); const pslOnly = new Set(); - domainData.pslTlds.forEach( domain => pslOnly.add(domain)); + Object.keys(domainData.pslTlds).forEach( domain => pslOnly.add(domain)); domainData.icannTlds.forEach( domain => pslOnly.delete(domain)); ctx.body = await ctx.render('domains/icann-vs-psl.hbs', { @@ -102,14 +100,6 @@ domainRouter.get('/domains/icann-vs-wsw.html', async (ctx:any) => { }); }); -domainRouter.get('/domains/psl-tlds.html', async (ctx:any) => { - ctx.redirect(ctx, '/psl/index.html'); -}); - -domainRouter.get('/psl/', async (ctx: any) => { - ctx.redirect(ctx, '/psl/index.html'); -}); - domainRouter.get('/domains/publicsuffix.html', async (ctx: any) => { ctx.redirect(ctx, '/psl/test.html'); }); @@ -118,40 +108,8 @@ domainRouter.post('/domains/publicsuffix.html', async (ctx: any) => { ctx.redirect(ctx, `/psl/test.html?q=${encodeURIComponent(ctx.request.body.hostname)}`); }); -domainRouter.get('/psl/index.html', async (ctx: any) => { - ctx.body = await ctx.render('psl/index.hbs', { - domains: domainData.pslTlds, - title: 'Public Suffix List Top Level Domains', - }); -}); - -domainRouter.get('/psl/test.html', async (ctx:any) => { - - const q = ctx.request.query.q; - const hostname = q; - let get:string|null|undefined; - let parsed:psl.ParsedDomain|psl.ParseError|null|undefined; - if (hostname) { - if (!psl.isValid(hostname)) { - ctx.flash('error', `${Handlebars.escapeExpression(hostname)} is not a valid public domain!`); - } else { - get = psl.get(hostname); - parsed = psl.parse(hostname); - - if (parsed.error) { - ctx.flash('error', `${parsed.error.message} for ${Handlebars.escapeExpression(hostname)}`); - parsed = null; - } - } - } - - ctx.body = await ctx.render('psl/test.hbs', { - examples: ['test.github.io', 'github.io'], - get, - parsed, - q, - title: 'Public Suffix Test', - }); +domainRouter.get('/domains/psl-tlds.html', async (ctx:any) => { + ctx.redirect(ctx, '/psl/index.html'); }); domainRouter.get('/domains/usable-tlds.html', async (ctx:any) => { diff --git a/src/routers/pslRouter.ts b/src/routers/pslRouter.ts new file mode 100644 index 0000000..8e5a620 --- /dev/null +++ b/src/routers/pslRouter.ts @@ -0,0 +1,70 @@ +import Handlebars from 'handlebars'; +import KoaRouter from 'koa-router'; +import path from 'path'; +import * as psl from 'psl'; + +import { ChangeLog } from '../changelog'; +import { ChangeLogUI } from '../ChangeLogUI'; +import * as domainData from '../data/domainData'; + +const pslRouter = new KoaRouter(); + +pslRouter.get('/psl/', async (ctx: any) => { + ctx.redirect(ctx, '/psl/index.html'); +}); + + +pslRouter.get('/psl/index.html', async (ctx: any) => { + + ctx.body = await ctx.render('psl/index.hbs', { + domains: Object.keys(domainData.pslTlds).sort((a, b) => { return a.localeCompare(b); }), + psl: domainData.pslTlds, + title: 'Public Suffix List Top Level Domains', + total: Object.values(domainData.pslTlds).reduce((acc, val) => acc + val.length, 0), + }); +}); + +pslRouter.get('/psl/test.html', async (ctx: any) => { + + const q = ctx.request.query.q; + const hostname = q; + let get: string | null | undefined; + let parsed: psl.ParsedDomain | psl.ParseError | null | undefined; + if (hostname) { + if (!psl.isValid(hostname)) { + ctx.flash('error', `${Handlebars.escapeExpression(hostname)} is not a valid public domain!`); + } else { + get = psl.get(hostname); + parsed = psl.parse(hostname); + + if (parsed.error) { + ctx.flash('error', `${parsed.error.message} for ${Handlebars.escapeExpression(hostname)}`); + parsed = null; + } + } + } + + ctx.body = await ctx.render('psl/test.hbs', { + examples: ['test.github.io', 'github.io'], + get, + parsed, + q, + title: 'Public Suffix Test', + }); +}); + +const pslChangeLogUI:ChangeLogUI = new ChangeLogUI( + new ChangeLog(path.join(__dirname, '../../data/publicsuffix/deltas')), + '/psl/index.html', + '/psl/changelog', + 'Public Suffix List', + 'https://botsin.space/@PublicSuffixChanges', +); +const pslChangeLogRouter = pslChangeLogUI.changelogRouter; +const pslChangeLogGetUrls = pslChangeLogUI.getUrls; + +export { + pslRouter, + pslChangeLogRouter, + pslChangeLogGetUrls, +} diff --git a/src/routers/tldsRouter.ts b/src/routers/tldsRouter.ts index a0a7ae6..2266521 100644 --- a/src/routers/tldsRouter.ts +++ b/src/routers/tldsRouter.ts @@ -1,8 +1,12 @@ import * as punycode from 'punycode'; import Router from 'koa-router'; +import path from 'path'; import * as wsw from 'whoisserver-world' import * as rdapData from '../data/rdapData'; +import { ChangeLog } from '../changelog'; +import { ChangeLogUI } from '../ChangeLogUI'; +import * as domainData from '../data/domainData'; const tldsRouter = new Router(); @@ -72,6 +76,7 @@ tldsRouter.get('/tlds/:tld/index.html', async (ctx:any) => { tldInfo.unicode = punycode.toUnicode(tld); ctx.body = await ctx.render('_tlds/index.hbs', { + publicSuffixes: domainData.pslTlds[tld], rdapUnofficial, sampleDomains, title: `Top Level Domain "${tld}"`, @@ -92,7 +97,19 @@ function getUrls():string[] { return retVal; } +const tldChangeLogUI: ChangeLogUI = new ChangeLogUI( + new ChangeLog(path.join(__dirname, '../../data/icann/deltas')), + '/tlds/index.html', + '/tlds/changelog', + 'ICANN TLD', + 'https://botsin.space/@TLDChanges', +); +const tldsChangeLogRouter = tldChangeLogUI.changelogRouter; +const tldsChangeLogGetUrls = tldChangeLogUI.getUrls; + export { tldsRouter, - getUrls + getUrls, + tldsChangeLogRouter, + tldsChangeLogGetUrls, } diff --git a/src/server.ts b/src/server.ts index 815b30b..915cd17 100644 --- a/src/server.ts +++ b/src/server.ts @@ -17,7 +17,6 @@ import * as transliteration from 'transliteration'; import config from './config'; import * as asn from './data/maxmindData'; import { resolverRouter } from './routers/resolverRouter'; -import { pslChangelogRouter, tldChangelogRouter } from './routers/changelogRouter'; import { domainRouter } from './routers/domainRouter'; import { httpRouter } from './routers/httpRouter'; import { ipRouter } from './routers/ipRouter'; @@ -25,9 +24,10 @@ import { logger , options as loggerOptions } from './logger'; import { cryptoRouter } from './routers/cryptoRouter'; import { datagenRouter } from './routers/datagenRouter'; import { dnsRouter } from './routers/dnsRouter'; +import { pslRouter, pslChangeLogRouter } from './routers/pslRouter'; import * as resolvers from './data/resolverData'; import { rootRouter } from './routers/rootRouter'; -import { tldsRouter } from './routers/tldsRouter'; +import { tldsRouter, tldsChangeLogRouter } from './routers/tldsRouter'; import { infoRouter } from './routers/infoRouter'; import * as domains from './data/domainData'; import * as rdapData from './data/rdapData'; @@ -124,6 +124,9 @@ app.use(KoaViews(path.join(__dirname, '..', 'views'), { return result; }, fromPunycode: function(domain:string) { return domain ? punycode.toUnicode(domain) : '(null)'; }, + get: function (map: { [key: string]: any}, key:string) { + return map[key]; + }, ifDomainUnicode: function(context:any, options:any) { return context && !context.match(/^[a-z]+$/) ? options.fn(this) : options.inverse(this); }, integerFormat: function(theNumber:any):string { if (theNumber == 0) { @@ -140,6 +143,9 @@ app.use(KoaViews(path.join(__dirname, '..', 'views'), { isArray: function(target:any) { return target && Array.isArray(target); }, + length: function(target:any) { + return target ? target.length : 0; + }, noCacheDev: function(multiParam:boolean) { if (process.env.COMMIT) { return ''; @@ -221,8 +227,9 @@ app.use(domainRouter.routes()); app.use(httpRouter.routes()); app.use(ipRouter.routes()); app.use(infoRouter.routes()); -app.use(pslChangelogRouter.routes()); -app.use(tldChangelogRouter.routes()); +app.use(pslChangeLogRouter.routes()); +app.use(pslRouter.routes()); +app.use(tldsChangeLogRouter.routes()); app.use(tldsRouter.routes()); async function main() { diff --git a/views/_tlds/index.hbs b/views/_tlds/index.hbs index 68f05e9..0795042 100644 --- a/views/_tlds/index.hbs +++ b/views/_tlds/index.hbs @@ -75,7 +75,7 @@
{{sampleDomains.length}} {{#each sampleDomains}} - {{this}}
+ {{this}}
{{/each}}
{{else}} @@ -83,6 +83,17 @@ {{/if}} + + Public Suffixes + +
+ {{length publicSuffixes}} + {{#each publicSuffixes}} + {{this}}
+ {{/each}} +
+ + Links diff --git a/views/psl/index.hbs b/views/psl/index.hbs index 75c3ad2..dc162ea 100644 --- a/views/psl/index.hbs +++ b/views/psl/index.hbs @@ -10,7 +10,7 @@
- Count: {{domains.length}} + Count: {{integerFormat domains.length}} top-level domains, {{integerFormat total}} public suffixes
Test @@ -18,10 +18,25 @@
-
    -{{#each domains}} -
  • {{this}}{{#ifDomainUnicode this}} ({{toPunycode this}}){{/ifDomainUnicode}}
    • -{{/each}} -
- + + + + + + + + + {{#each domains}} + + + + + {{/each}} + +
Top Level DomainPublic Suffixes
{{this}}{{#ifDomainUnicode this}} ({{toPunycode this}}){{/ifDomainUnicode}}
+ {{length (get ../psl this) }} + {{#each (get ../psl this)}} + {{this}}
+ {{/each}} +
{{> below}} diff --git a/views/tlds/index.hbs b/views/tlds/index.hbs index caa9cbb..e04c498 100644 --- a/views/tlds/index.hbs +++ b/views/tlds/index.hbs @@ -6,7 +6,7 @@
- Count: {{tlds.length}} + Count: {{integerFormat tlds.length}}
ChangeLog