Move docs into the database. Add doc searching, and migrate to bootstrap4.

Author: dpage

docs/__init__.py

@@ -0,0 +1,25 @@
+##########################################################################
+#
+# pgAdmin Website
+#
+# Copyright (C) 2017, The pgAdmin Development Team
+# This software is released under the PostgreSQL Licence
+#
+##########################################################################
+
+from __future__ import unicode_literals
+
+from django.contrib import admin
+
+from .models import Page
+
+
+class PageAdmin(admin.ModelAdmin):
+    list_display = ('_get_title',)
+
+    def _get_title(self, obj):
+        return str(obj)
+    _get_title.allow_tags = True
+
+
+admin.site.register(Page, PageAdmin)

docs/admin.py

@@ -0,0 +1,16 @@
+##########################################################################
+#
+# pgAdmin Website
+#
+# Copyright (C) 2017, The pgAdmin Development Team
+# This software is released under the PostgreSQL Licence
+#
+##########################################################################
+
+from __future__ import unicode_literals
+
+from django.apps import AppConfig
+
+
+class DocsConfig(AppConfig):
+    name = 'docs'

docs/apps.py

@@ -0,0 +1,166 @@
+##########################################################################
+#
+# pgAdmin Website
+#
+# Copyright (C) 2017, The pgAdmin Development Team
+# This software is released under the PostgreSQL Licence
+#
+##########################################################################
+
+from bs4 import BeautifulSoup
+import glob
+import os
+from django.core.management.base import BaseCommand, CommandError
+from docs.models import Page
+from download.models import Package, Version
+from django.db.utils import IntegrityError
+from django.db.transaction import set_autocommit, commit, rollback
+from pgaweb.settings import STATIC_URL
+
+
+class Command(BaseCommand):
+    help = 'Load HTML docs from a directory into the database.'
+
+    def add_arguments(self, parser):
+        parser.add_argument('package', help="the name of the Package to link "
+                                            "the document to.")
+
+        parser.add_argument('version', help="the version number of the "
+                                            "package to link the document to. "
+                                            "Any text including and following "
+                                            "the first non-numeric or . will "
+                                            "be treated as the suffix.")
+
+        parser.add_argument('source', help="the path to the directory "
+                                           "containing the HTML files to load "
+                                           "and index. Must be relative to "
+                                           "$INSTDIR/static/docs")
+
+    def load_docs(self, directory, static, version):
+        """
+        Load document pages from the given directory into the Document
+
+        :param directory: The directory to load pages from
+        :param directory: The path to static files
+        :param version: The Version object to load Pages into
+        :return: The number of Pages loaded
+        """
+
+        count = 0
+        path = os.path.join(directory, '**/*.htm*')
+
+        for file in glob.iglob(path, recursive=True):
+            self.stdout.write("Loading: {} ".format(file))
+
+            f = open(file, 'r')
+            html = f.read()
+            f.close()
+
+            # Get the filename
+            filename = file.replace(directory, '')
+            if filename.startswith('/'):
+                filename = filename[1:]
+
+            static_dir = os.path.join(STATIC_URL, static)
+            if '/' in filename:
+                static_dir = os.path.join(STATIC_URL, static,
+                                          filename[0:filename.rfind('/')])
+
+            # Extract the HTML
+            soup = BeautifulSoup(html, features="html.parser")
+
+            # Update paths to images
+            for img in soup.findAll('img'):
+                img['src'] = os.path.join(static_dir, img['src'])
+
+            for obj in soup.findAll('object'):
+                if obj.has_attr('data'):
+                    obj['data'] = os.path.join(static_dir, obj['data'])
+                if obj.has_attr('width'):
+                    del obj['width']
+
+            # Remove the search box if present
+            for div in soup.findAll('div'):
+                if div.has_attr('id') and div['id'] == 'searchbox':
+                    div.extract()
+
+            # If there's no title, use the filename (without the .html)
+            if soup.find('title') is not None:
+                title = soup.find('title').decode_contents(formatter="html").strip()
+            else:
+                title = filename[:-5]
+
+            # Remove the title, as we no longer need it
+            [x.extract() for x in soup.find_all('title')]
+
+            # Get the page header navigation
+            header = soup.find('div', role='navigation').decode_contents(formatter="html").strip()
+
+            # Get the contents panel
+            contents_soup = soup.find('div', class_='sphinxsidebarwrapper')
+            [x.extract() for x in contents_soup.find_all('h3')]
+            contents = contents_soup.decode_contents(formatter="html").strip()
+
+            # Get the page content
+            body = soup.find('div', class_='body').decode_contents(formatter="html").strip()
+
+            # We want the Contents header to be an H1 not H2
+            contents = contents.replace('h3>', 'h1>')
+
+            # Create the new Page
+            try:
+                page = Page.objects.create(version=version, file=filename,
+                                           title=title, header=header,
+                                           contents=contents, body=body)
+            except IntegrityError as e:
+                rollback()
+                raise CommandError("Rolling back: failed to insert {}: {}".
+                                   format(filename, str(e)))
+
+            page.save()
+
+            count = count + 1
+
+        return count
+
+    def handle(self, *args, **options):
+        """The core structure of the app."""
+
+        # Get the various paths we need
+        root_dir = os.path.realpath(os.path.join(
+            os.path.dirname(__file__), '../../../'))
+
+        source_path = os.path.join(root_dir, 'static/docs', options['source'])
+
+        static_dir = os.path.join('docs', options['source'])
+        if os.path.isfile(source_path):
+            static_dir = os.path.dirname(os.path.join('docs',
+                                                      options['source']))
+
+        # Disable autocommit so this all runs in a transaction
+        set_autocommit(False)
+
+        # Check the Package exists
+        package = Package.objects.filter(name=options['package']).first()
+
+        if package is None:
+            raise CommandError("The Package object could not be found.")
+
+        # Check the Version exists
+        version = Version.objects.filter(name=options['version'],
+                                         package=package).first()
+
+        if version is None:
+            raise CommandError("The Version object could not be found.")
+
+        # Delete all existing pages and load the new ones from the dir
+        Page.objects.filter(version_id=version.id).delete()
+
+        count = self.load_docs(source_path,
+                               static_dir,
+                               version)
+
+        commit()
+
+        self.stdout.write("Loaded {} pages into {}".format(
+            count, str(version)))

docs/management/__init__.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.18 on 2019-07-18 11:04
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import tsvector_field
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('download', '0004_auto_20190718_0929'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Page',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('file', models.CharField(help_text='The file name of the page', max_length=100)),
+                ('title', models.CharField(help_text='The title of the page', max_length=500)),
+                ('header', models.TextField(blank=True, help_text='The header tabs for the page.')),
+                ('body', models.TextField(blank=True, help_text='The body of the page.')),
+                ('search', tsvector_field.SearchVectorField(blank=True, columns=[tsvector_field.WeightedColumn('title', 'A'), tsvector_field.WeightedColumn('body', 'D')], language='english')),
+                ('version', models.ForeignKey(help_text='Select the Product Version to which this page applies.', on_delete=django.db.models.deletion.CASCADE, to='download.Version')),
+            ],
+        ),
+        migrations.AlterUniqueTogether(
+            name='page',
+            unique_together=set([('version', 'file')]),
+        ),
+    ]

docs/management/commands/__init__.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.18 on 2019-07-19 09:42
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('docs', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='page',
+            options={'ordering': ('version', 'title')},
+        ),
+        migrations.AddField(
+            model_name='page',
+            name='contents',
+            field=models.TextField(blank=True, help_text='The contents block of the page.'),
+        ),
+        migrations.AlterField(
+            model_name='page',
+            name='header',
+            field=models.TextField(blank=True, help_text='The header block for the page.'),
+        ),
+    ]

docs/management/commands/docloader.py

@@ -0,0 +1,59 @@
+##########################################################################
+#
+# pgAdmin Website
+#
+# Copyright (C) 2017, The pgAdmin Development Team
+# This software is released under the PostgreSQL Licence
+#
+##########################################################################
+
+from django.core.cache import cache
+from django.db import models
+from django.db.models.signals import post_save
+from django.dispatch import receiver
+from django.urls import reverse
+from download.models import Version
+import tsvector_field
+
+
+@receiver(post_save)
+def clear_the_cache(**kwargs):
+    if kwargs['sender']._meta.label != 'admin.LogEntry':
+        cache.clear()
+
+
+class Page(models.Model):
+    """
+    Stores documentation pages that are not external files. Related to
+    :model:`docs.Document`,
+    """
+    version = models.ForeignKey(Version, on_delete=models.CASCADE,
+                                help_text="Select the Product Version to "
+                                          "which this page applies.")
+    file = models.CharField(null=False, blank=False, max_length=100,
+                            help_text="The file name of the page")
+    title = models.CharField(null=False, blank=False, max_length=500,
+                             help_text="The title of the page")
+    header = models.TextField(null=False, blank=True,
+                              help_text="The header block for the page.")
+    contents = models.TextField(null=False, blank=True,
+                                help_text="The contents block of the page.")
+    body = models.TextField(null=False, blank=True,
+                            help_text="The body of the page.")
+    search = tsvector_field.SearchVectorField([
+        tsvector_field.WeightedColumn('title', 'A'),
+        tsvector_field.WeightedColumn('body', 'D'),
+    ], 'english', blank=True)
+
+    @property
+    def url(self):
+        return reverse('page', args=[self.version.package.slug,
+                                     self.version.name,
+                                     self.file])
+
+    class Meta:
+        unique_together = ['version', 'file']
+        ordering = ('version', 'title')
+
+    def __str__(self):
+        return self.title

docs/migrations/0001_initial.py

@@ -0,0 +1,25 @@
+{% extends "pgaweb/page.html" %}
+
+{% block title %}Documentation{% endblock %}
+
+{% block content %}
+<h1>Documentation</h1>
+
+<p>The pgAdmin documentation for the current development code, and recent major releases of the application is available
+    for online browsing. Please select the documentation version you would like to view from the options below.</p>
+
+<p>The documentation is automatically imported from the pgAdmin <a href="{% url 'development_resources' %}#git">GIT</a>
+    source code repository, and is only available in English.</p>
+
+{% regroup pages by version.package.name as page_list %}
+{% for page in page_list %}
+<h2>{{ page.grouper }}</h2>
+<div class="well pga-downloads">
+  <ul class="fa-ul">
+  {% for pg in page.list %}
+  <li><i class="fa-li fa fa-book" aria-hidden="true"></i><a href="{% url 'page' pg.version.package.slug pg.version.slug 'index.html' %}">{{ pg.version.name }}</a>{% if not pg.version.pre_release %} (released {{ pg.version.released }}){% endif %}</li>
+  {% endfor %}
+  </ul>
+</div>
+{% endfor %}
+{% endblock %}