Hugo Docs Modification (linode#1127)

Hugo Docs Modification

Author: GuessWhoSamFoo

docs/.gitignore renamed to .gitignore

@@ -1,3 +1,10 @@
+__pycache__/
+.cache/
+.scrapy
+.pynb_checkpoints
+.env
+.venv
+
 .DS_Store
 .DS_Store?
 ._*

.travis.yml

@@ -0,0 +1,32 @@
+language: python
+cache:
+    pip: true
+    directories:
+    - node_modules
+matrix:
+    include:
+    - python: "3.6"
+
+before_install:
+    - echo "Installing Hugo 0.30.2"
+#    - curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash -
+#    - sudo apt install nodejs
+    - wget https://github.com/gohugoio/hugo/releases/download/v0.30.2/hugo_0.30.2_Linux-64bit.deb
+    - sudo dpkg -i hugo*.deb
+    - hugo version
+#    - sudo npm install --global gulp-cli
+#    - npm install
+
+install:
+    - echo "Install pytest and pip"
+    - pip install -r ci/requirements.txt
+
+before_script:
+    - echo "Setting up local development server"
+    - hugo server &
+    - sleep 7
+
+script:
+    - echo "Checking for style guidelines"
+    - python -m pytest -n 2
+

archetypes/content.md

@@ -0,0 +1,59 @@
+---
+author:
+  name: Linode Community
+  email: docs@linode.com
+description: 'Two to three sentences describing your guide.'
+keywords: ['list','of','keywords','and key phrases']
+license: '[CC BY-ND 4.0](https://creativecommons.org/licenses/by-nd/4.0)'
+published: {{ .Date }}
+modified: {{ .Date }}
+expiry_date: {{ .Date }}
+deprecated: {{ .Date > $.Page.Params.expiry_date }}
+modified_by:
+  name: Linode
+title: "{{ replace .TranslationBaseName "-" " " | title }}"
+contributor:
+  name: Your Name
+  link: Github/Twitter Link
+external_resources:
+- '[Link Title 1](http://www.example.com)'
+- '[Link Title 2](http://www.example.net)'
+---
+
+*This is a Linode Community guide. If you're an expert on something for which we need a guide, you too can [get paid to write for us](/docs/contribute).*
+
+----
+
+<!---  See https://www.linode.com/docs/style-guide for more details on guide formatting.  --->
+
+
+## Before You Begin
+
+1.  Familiarize yourself with our [Getting Started](/docs/getting-started) guide and complete the steps for setting your Linode's hostname and timezone.
+
+2.  This guide will use `sudo` wherever possible. Complete the sections of our [Securing Your Server](/docs/security/securing-your-server) to create a standard user account, harden SSH access and remove unnecessary network services. Do **not** follow the Configure a Firewall section yet--this guide includes firewall rules specifically for an OpenVPN server.
+
+3.  Update your system:
+
+        sudo apt-get update && sudo apt-get upgrade
+
+<!-- Include one of the following notes if appropriate. --->
+
+{{< note >}}
+This guide is written for a non-root user. Commands that require elevated privileges are prefixed with `sudo`. If you’re not familiar with the `sudo` command, see the [Users and Groups](/docs/tools-reference/linux-users-and-groups) guide.
+{{< /note >}}
+
+{{< note >}}
+The steps in this guide require root privileges. Be sure to run the steps below as `root` or with the `sudo` prefix. For more information on privileges, see our [Users and Groups](/docs/tools-reference/linux-users-and-groups) guide.
+{{< /note >}}
+
+
+{{< caution >}}
+Highlight warnings that could adversely affect a user's system with the Caution style.
+{{< /caution >}}
+
+{{< file "/etc/hosts" aconf >}}
+192.0.2.0/24      # Sample IP addresses
+198.51.100.0/24
+203.0.113.0/24
+{{< /file >}}

archetypes/default.md

@@ -0,0 +1,6 @@
+---
+title: "{{ replace .TranslationBaseName "-" " " | title }}"
+date: {{ .Date }}
+draft: true
+---
+

ci/README.md

@@ -0,0 +1,18 @@
+Running on a virtual environment is recommended.
+
+Install pip requirments:
+
+    `pip install -r requirements.txt`
+
+Start the Hugo development server. Runs on `localhost:1313/docs/` by default:
+
+    `hugo server`
+
+Run the test environment from the root directory. Python 3 is required.
+
+    `python -m pytest`
+
+To use multiple cores when running tests, add `-n` (2 is enough for current tests):
+
+    `python -m pytest -n <Additional Cores>`
+

ci/conftest.py

@@ -0,0 +1,47 @@
+import pytest
+import os
+import fnmatch
+from functools import wraps
+
+
+LOCALHOST = "http://localhost:1313/docs/"
+
+def file_io(func):
+    """
+    Open and close file objects for tests
+    """
+    @wraps(func)
+    def wrapper(md_filepaths):
+        # Before test
+        f = open(md_filepaths, 'r')
+        # After test
+        r = func(f)
+        f.close()
+        return r
+    return wrapper
+
+
+@pytest.fixture(scope='module', autouse=True)
+def md_index(path='.', extension='*.md'):
+    """
+    Traverses root directory
+    """
+    index = []
+    exclude = ['node_modules']
+    for root, dirnames, filenames in os.walk(path):
+        dirnames[:] = [d for d in dirnames if d not in exclude]
+        for filename in fnmatch.filter(filenames, extension):
+            index.append(os.path.join(root, filename))
+    return index
+
+
+@pytest.fixture(params=md_index())
+def md_filepaths(request):
+    return request.param
+
+
+#@pytest.fixture(params=[open(i) for i in md_index()])
+#def md_files(request):
+#    return request.param
+
+

ci/requirements.txt

@@ -0,0 +1,5 @@
+regex>=2017.4.5
+requests>.2.14.2
+scrapy>=1.4.0
+pytest-xdist>=1.20.1
+PyYAML>=3.12

ci/test_404.py

@@ -0,0 +1,60 @@
+import pytest
+
+import logging
+import scrapy
+from scrapy import Item, Field
+from scrapy.linkextractors import LinkExtractor
+from scrapy.spiders import CrawlSpider, Rule
+from scrapy.crawler import CrawlerProcess
+
+class Docs404Item(Item):
+    referer = Field()
+    status = Field()
+    url = Field()
+
+class Docs404Spider(CrawlSpider):
+
+    def __init__(self, *args, **kwargs):
+        loggers = ['scrapy.core.engine',
+                   'scrapy.downloadermiddlewares.redirect',
+                   'scrapy.spidermiddlewares.offsite',
+                   'scrapy.middleware']
+        for l in loggers:
+            logger = logging.getLogger(l)
+            logger.setLevel(logging.WARNING)
+        super().__init__(*args, **kwargs)
+
+#    Delay if server is returning lots of 500s
+#    DOWNLOAD_DELAY=0.1
+    name = 'docs404'
+    allowed_domains = ['localhost' ]
+    start_urls = ['http://localhost:1313/docs']
+    handle_httpstatus_list = [404]
+
+    rules = (
+        Rule(LinkExtractor(allow=r'/docs/', deny=r'/docs/contribute'),
+             callback='parse_item', follow=True),
+    )
+
+    def parse_item(self, response):
+        item = Docs404Item()
+
+        if response.status == 404:
+            item['referer'] = response.request.headers.get('Referer')
+            item['status'] = response.status
+            item['url'] = response.url
+            return item
+
+
+def test_404():
+    import os
+    process = CrawlerProcess({ 'USER_AGENT': 'docs404',
+                               'FEED_URI': 'temp.csv',
+                               'FEED_FORMAT': 'csv' })
+    process.crawl(Docs404Spider)
+    process.start()
+    f = open('temp.csv')
+    os.remove('temp.csv')
+    assert sum([1 for line in f]) == 1,'404 response in HTML - see scraper logs'
+
+

ci/test_alias.py

@@ -0,0 +1,43 @@
+import pytest
+
+from contextlib import redirect_stdout
+import re
+import ast
+from datetime import datetime
+import urllib.request
+from urllib.error import HTTPError, URLError
+from conftest import file_io, LOCALHOST
+
+
+@file_io
+def test_alias(md_filepaths):
+    aliases = []
+    valid_alias = True
+    for line in md_filepaths:
+        # Check if file is expired
+        today = datetime.now()
+        has_exp_header = re.match(r'^expiryDate: ', line)
+        if has_exp_header:
+            exp_date = datetime.strptime(line.split()[1], '%Y-%m-%d')
+            if exp_date <= today:
+                pytest.skip("Marked expired not checked")
+
+        match = re.match(r'^aliases: \[.*\]', line)
+        if match:
+            new_line = match.group()
+            # Literal evaluation of brackets in alias
+            aliases += ast.literal_eval(new_line[new_line.find("["):])
+
+    #Check case where alias points back to URL
+    if md_filepaths.name.lstrip('./content/')[:-3] in \
+            [a.rstrip('/') for a in aliases]:
+        valid_alias = False
+        print('Circular alias: ' + path)
+    for alias in aliases:
+        try:
+            urllib.request.urlopen(LOCALHOST + alias).getcode()
+        except HTTPError:
+            valid_alias = False
+            print('404 alias: ' + alias)
+    assert valid_alias == True,'Not a valid alias'
+

ci/test_filename.py

@@ -0,0 +1,7 @@
+import pytest
+
+def test_filename(md_filepaths):
+    if any(e in md_filepaths for e in ['README.md', 'CHANGELOG.md']):
+        assert True
+    else:
+        assert md_filepaths.islower() == True,'Filename should be lowercase'

ci/test_whitespace.py

@@ -0,0 +1,17 @@
+import pytest
+
+from contextlib import redirect_stdout
+import regex
+from conftest import file_io
+
+@file_io
+def test_whitespace(md_filepaths):
+    has_whitespace = False
+    for line_number, line in enumerate(md_filepaths, 1):
+        match = regex.search(r'[\t]+$', line)
+        if match:
+            has_whitespace = True
+            print("Trailing whitespace at " + str(line_number) + \
+                  ":" + str(match.start()))
+    assert has_whitespace == False
+

ci/test_yaml.py

@@ -0,0 +1,36 @@
+import pytest
+
+import regex
+from dateutil.parser import parse
+import sys
+import yaml
+import json
+from conftest import file_io
+
+with open('ci/yaml_rules.json') as json_data:
+    requirements = json.load(json_data)
+
+
+@file_io
+def test_yaml(md_filepaths):
+    for filestring in md_filepaths:
+        reg = regex.compile(r'^---(.*?)---',flags=regex.DOTALL)
+        match = regex.search(reg, filestring)
+        # Hack until README.md files won't be passed in
+        # assert match
+        if not match: return
+        yaml_text = match.group(1)
+        parsed_yaml = yaml.load(yaml_text)
+        for requirement in requirements:
+            req = requirements[requirement]
+            if req['required']:
+                assert requirement in parsed_yaml, 'YAML metadata missing required element: ' + requirement
+            if req['type'] is 'link':
+                regexp = regex.compile(r'\[(.*)\]\((.*)\)')
+                assert regex.match(regexp,parsed_yaml[requirement]), 'YAML metadata formatting error: ' + requirement
+            if req['type'] is 'date':
+                try:
+                    parse(str(parsed_yaml[requirement]))
+                except ValueError:
+                    assert False, 'YAML metadata formatting error: ' + requirement + ' date parse failed.'
+

ci/yaml_rules.json

@@ -0,0 +1,45 @@
+{
+    "author": {
+        "required": true,
+        "elements":["name","email"],
+        "type":"dict"
+        },
+    "title": {
+        "required": true,
+        "elements": false,
+        "type":"text"
+        },
+    "description": {
+        "required": true,
+        "elements": false,
+        "type":"text"
+        },
+    "keywords": {
+        "required": true,
+        "elements":"list",
+        "type":"list"
+        },
+    "license": {
+        "required": true,
+        "elements": false,
+        "type":"link"
+        },
+    "modified": {
+        "required": true,
+        "elements": false,
+        "type":"date"
+        },
+    "modified_by": {
+        "required": true,
+        "elements": false,
+        "type":"text"
+        },
+    "published": {
+        "required": true,
+        "type":"date",
+        "elements": false
+        }
+}
+
+
+