Skip to content

Commit c3587bd

Browse files
abessiariabessiari
committed
added ssl/qos suport
1 parent 81373e6 commit c3587bd

19 files changed

+211
-585
lines changed

‎docker-vars.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,4 @@ repo_map:
99

1010
tag: latest
1111
docker_hub_user: "{{ lookup('env', 'USER')|lower }}"
12-
apache_proxy_image: geneontology/apache-proxy:v2
12+
apache_proxy_image: geneontology/apache-proxy:v3

‎files/qos-noop.conf

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
<IfModule qos_module>
2+
# minimum request rate (bytes/sec at request reading):
3+
#QS_SrvRequestRate 120
4+
5+
# limits the connections for this virtual host:
6+
#QS_SrvMaxConn 100
7+
8+
# allows keep-alive support till the server reaches 600 connections:
9+
#QS_SrvMaxConnClose 600
10+
11+
# allows max 50 connections from a single ip address:
12+
#QS_SrvMaxConnPerIP 50
13+
</IfModule>

‎production/README.md

+8-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# Noctua Production Deployment
22

3+
34
This repository enables the deployment of the noctua stack to AWS. It includes
45
minerva, barista, and noctua and it points to an external amigo instance.
56

@@ -8,14 +9,18 @@ minerva, barista, and noctua and it points to an external amigo instance.
89
- vars.yaml
910
- docker-vars.yaml
1011
- s3-vars.yaml
12+
- ssl-vars.yaml
1113
- stage.yaml
14+
- qos-vars.yaml
1215
- start_services.yaml
1316

1417
## Artifacts Deployed To Staging directory On AWS:
1518
- blazegraph.jnl
1619
- Cloned repositories:
1720
- noctua-form, noctua-landing-page, noctua-models, go-site and noctua-visual-pathway-editor.
1821
- s3 credentials used to push apache logs to s3 buckets
22+
- s3 credentials used to download ssl credentials from s3 buckets
23+
- qos.conf and robots.txt for apache mitigation
1924
- github OAUTH client id and secret
2025
- docker-production-compose and various configuration files from template directory
2126

@@ -79,7 +84,9 @@ Check list:
7984
- [ ] <b>Make DNS names for barista and noctua point to public ip address on AWS Route 53.</b>
8085
- [ ] Location of SSH keys need to be replaced after copying config-stack.yaml.sample
8186
- [ ] Github credentials will need to be replaced in config-stack.yaml.sample
82-
- [ ] s3 credntials are placed in a file using format described above
87+
- [ ] s3 credentials are placed in a file using format described above
88+
- [ ] s3 uri if ssl is enabled. Location of ssl certs/key
89+
- [ ] qos mitigation if qos is enabled
8390
- [ ] Location of blazegraph.jnl. This assumes you have generated the journal using steps above
8491
- [ ] Use same workspace name as in previous step
8592
- [ ] Remember you can use the -dry-run option

‎production/config-instance.yaml.sample

+1
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,5 @@ instance:
99
open_ports:
1010
- 80
1111
- 22
12+
- 443
1213
disk_size: 100

‎production/config-stack.yaml.sample

+10-5
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,11 @@ stack:
1414
docker_hub_user: geneontology
1515
tag: v2
1616

17-
# For production the proxy will listen on port 80
18-
noctua_proxy_port: 80
19-
2017
# DNS hostname for noctua
2118
noctua_host: aes-test-noctua.geneontology.io
22-
noctua_lookup_url: http://aes-test-noctua.geneontology.io
2319

2420
# DNS hostname for barista
2521
barista_lookup_host: aes-test-barista.geneontology.io
26-
barista_lookup_url: http://aes-test-barista.geneontology.io
2722

2823
# url for golr
2924
golr_lookup_url: http://noctua-golr.berkeleybop.org
@@ -32,6 +27,16 @@ stack:
3227
S3_CRED_FILE: REPLACE_ME
3328
S3_BUCKET: go-service-logs
3429

30+
# Used to download ssl credentials from s3 buckets. (full s3 uri .tar.gz)
31+
USE_SSL: 1
32+
S3_SSL_CRED_FILE: REPLACE_ME
33+
S3_SSL_CERTS_LOCATION: REPLACE_ME
34+
35+
# Enable QOS
36+
USE_QOS: 1
37+
QS_ClientEventBlockCount: "350 300"
38+
QS_ClientEventBlockExcludeIP: "REPLACE_ME_ADDR1 REPLACE_ME_ADDR2"
39+
3540
# OAUTH
3641
github_client_id: 'REPLACE_ME'
3742
github_client_secret: 'REPLACE_ME'

‎qos-vars.yaml

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
---
2+
###############
3+
# download ssl certs
4+
################
5+
USE_QOS: 1
6+
QS_ClientEntries 200000
7+
QS_SrvMaxConnPerIP: 50
8+
QS_ClientEventPerSecLimit: 150
9+
QS_ClientEventRequestLimit: 150
10+
QS_ClientEventBlockCount: "350 300"
11+
QS_ClientEventBlockExcludeIP: "9.9.9.9"

‎ssl-vars.yaml

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
---
2+
###############
3+
# download ssl certs
4+
################
5+
USE_SSL: 0
6+
S3_SSL_CERTS_LOCATION: REPLACE_ME
7+
S3_SSL_CRED_FILE: REPLACE_ME
8+

‎stage.yaml

+33-9
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
- vars.yaml
44
- docker-vars.yaml
55
- s3-vars.yaml
6+
- ssl-vars.yaml
7+
- qos-vars.yaml
68

79
tasks:
810
- name: Create stage directories
@@ -13,13 +15,7 @@
1315
- conf
1416
- barista
1517
- httpd-confs
16-
17-
- name: install configs from templates directory
18-
template:
19-
src: '{{ item.file }}'
20-
dest: '{{ stage_dir }}/{{ item.dir }}'
21-
with_items:
22-
- { file: 'httpd.conf', dir: 'httpd-confs' }
18+
- credentials
2319

2420
- name: Check if repo is staged
2521
stat:
@@ -53,14 +49,42 @@
5349
- { file: 'httpd-vhosts-prod-noctua.conf', dir: 'httpd-confs' }
5450
- { file: 'github.yaml', dir: 'barista' }
5551
- { file: 'startup.yaml', dir: 'conf' }
52+
53+
- name: install ssl configs from templates directory
54+
template:
55+
src: '{{ item.file }}'
56+
dest: '{{ stage_dir }}/{{ item.dir }}'
57+
with_items:
58+
- { file: 'httpd-vhosts-prod-barista-ssl.conf', dir: 'httpd-confs' }
59+
- { file: 'httpd-vhosts-prod-noctua-ssl.conf', dir: 'httpd-confs' }
60+
when: USE_SSL | bool
61+
62+
- name: install qos config from templates directory
63+
template:
64+
src: qos.conf
65+
dest: '{{ stage_dir }}/qos.conf'
66+
when: USE_QOS | bool
67+
5668
- name: copy s3cfg
5769
copy:
5870
src: "{{ S3_CRED_FILE }}"
59-
dest: "{{ stage_dir }}/s3cfg"
71+
dest: "{{ stage_dir }}/credentials/s3cfg"
72+
73+
- name: copy s3cfg
74+
copy:
75+
src: "{{ S3_SSL_CRED_FILE }}"
76+
dest: "{{ stage_dir }}/credentials/ssl-s3cfg"
77+
78+
- name: copy noop qos.conf
79+
copy:
80+
src: "files/qos-noop.conf"
81+
dest: '{{ stage_dir }}/qos.conf'
82+
when: not USE_QOS | bool
83+
6084
- name: copy robots.txt
6185
copy:
6286
src: "files/robots.txt"
63-
dest: "{{ stage_dir }}/httpd-confs/robots.txt"
87+
dest: "{{ stage_dir }}/robots.txt"
6488
- name: copy blazegraph.jnl
6589
copy:
6690
src: "{{ BLAZEGRAPH_JOURNAL }}"

‎templates/docker-compose-production.yaml

+8-6
Original file line numberDiff line numberDiff line change
@@ -56,17 +56,19 @@ services:
5656
container_name: apache_noctua
5757
image: {{ apache_proxy_image }}
5858
volumes:
59-
- {{ stage_dir }}/httpd-confs/httpd-vhosts-prod-noctua.conf:/etc/apache2/sites-enabled/httpd-vhosts-noctua.conf
60-
- {{ stage_dir }}/httpd-confs/httpd-vhosts-prod-barista.conf:/etc/apache2/sites-enabled/httpd-vhosts-barista.conf
61-
- {{ stage_dir }}/httpd-confs/robots.txt:/var/www/html/robots.txt
59+
- {{ stage_dir }}/httpd-confs:/etc/apache2/sites-enabled
60+
- {{ stage_dir }}/qos.conf:/etc/apache2/mods-enabled/qos.conf
61+
- {{ stage_dir }}/robots.txt:/var/www/html/robots.txt
6262
- {{ stage_dir }}/apache_logs:/var/log/apache2
63-
- {{ stage_dir }}/s3cfg:/opt/credentials/s3cfg
63+
- {{ stage_dir }}/credentials:/opt/credentials
6464
ports:
65-
- "{{ noctua_proxy_port }}:80"
65+
- "80:80"
66+
- "443:443"
6667
environment:
67-
- USE_S3=1
6868
- S3_PATH={{ S3_PATH }}/noctua
6969
- S3_BUCKET={{ S3_BUCKET }}
70+
- USE_SSL={{ USE_SSL }}
71+
- S3_SSL_CERTS_LOCATION={{ S3_SSL_CERTS_LOCATION }}
7072
init: true
7173
restart: unless-stopped
7274
depends_on:

‎templates/httpd-vhosts-amigo.conf

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<VirtualHost *:80>
2-
ErrorLog "/var/log/apache2/amigo-error_log"
3-
CustomLog "/var/log/apache2/amigo-access_log" common
2+
ErrorLog "/var/log/apache2/amigo-error.log"
3+
CustomLog "/var/log/apache2/amigo-access.log" common
44
ProxyPass / http://amigo:8080/
55
ProxyPassReverse / http://amigo:8080/
66
RewriteEngine on

‎templates/httpd-vhosts-barista.conf

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<VirtualHost *:80>
2-
ErrorLog "/var/log/apache2/barista-error_log"
3-
CustomLog "/var/log/apache2/barista-access_log" common
2+
ErrorLog "/var/log/apache2/barista-error.log"
3+
CustomLog "/var/log/apache2/barista-access.log" common
44
ProxyPass / http://barista:3400/
55
ProxyPassReverse / http://barista:3400/
66
</VirtualHost>

‎templates/httpd-vhosts-golr.conf

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<VirtualHost *:80>
2-
ErrorLog "/var/log/apache2/solr-error_log"
3-
CustomLog "/var/log/apache2/solr-access_log" common
2+
ErrorLog "/var/log/apache2/solr-error.log"
3+
CustomLog "/var/log/apache2/solr-access.log" common
44
ProxyPass / http://golr:8080/
55
ProxyPassReverse / http://golr:8080/
66
RewriteEngine on

‎templates/httpd-vhosts-noctua.conf

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<VirtualHost *:80>
2-
ErrorLog "/var/log/apache2/noctua-error_log"
3-
CustomLog "/var/log/apache2/noctua-access_log" common
2+
ErrorLog "/var/log/apache2/noctua-error.log"
3+
CustomLog "/var/log/apache2/noctua-access.log" common
44
ProxyPass / http://noctua:8910/
55
ProxyPassReverse / http://noctua:8910/
66
</VirtualHost>

‎templates/httpd-vhosts-prod-barista-ssl.conf

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
<VirtualHost *:443>
2+
ServerAdmin admin@localhost
3+
ServerName {{ barista_lookup_host }}
4+
ServerAlias {{ barista_lookup_host_alias }}
5+
6+
Alias /robots.txt /var/www/html/robots.txt
7+
RewriteEngine On
8+
RewriteRule ^/robots.txt /robots.txt
9+
10+
## Get aggressive with badly behaving bots.
11+
RewriteCond %{HTTP_USER_AGENT} ^.*Adsbot.*$ [OR]
12+
RewriteCond %{HTTP_USER_AGENT} ^.*AhrefsBot.*$ [OR]
13+
RewriteCond %{HTTP_USER_AGENT} ^.*Amazonbot.*$ [OR]
14+
RewriteCond %{HTTP_USER_AGENT} ^.*Applebot.*$ [OR]
15+
RewriteCond %{HTTP_USER_AGENT} ^.*BingBot.*$ [OR]
16+
RewriteCond %{HTTP_USER_AGENT} ^.*DotBot.*$ [OR]
17+
RewriteCond %{HTTP_USER_AGENT} ^.*Googlebot.*$ [OR]
18+
RewriteCond %{HTTP_USER_AGENT} ^.*infotiger.*$ [OR]
19+
RewriteCond %{HTTP_USER_AGENT} ^.*MauiBot.*$ [OR]
20+
RewriteCond %{HTTP_USER_AGENT} ^.*PetalBot.*$ [OR]
21+
RewriteCond %{HTTP_USER_AGENT} ^.*semrush.*$ [OR]
22+
RewriteCond %{HTTP_USER_AGENT} ^.*WhatWeb.*$ [OR]
23+
RewriteCond %{HTTP_USER_AGENT} ^.*WhatWeb.*$
24+
RewriteRule . - [R=403,L]
25+
26+
27+
ErrorLog "/var/log/apache2/barista-error.log"
28+
CustomLog "/var/log/apache2/barista-access.log" common
29+
## Proxy.
30+
ProxyPreserveHost On
31+
ProxyRequests Off
32+
ProxyPass / http://barista:3400/
33+
ProxyPassReverse / http://barista:3400/
34+
35+
SSLEngine on
36+
SSLCertificateFile /opt/credentials/fullchain.pem
37+
SSLCertificateKeyFile /opt/credentials/privkey.pem
38+
</VirtualHost>

‎templates/httpd-vhosts-prod-barista.conf

+2-2
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@
2424
RewriteRule . - [R=403,L]
2525

2626

27-
ErrorLog "/var/log/apache2/barista-error_log"
28-
CustomLog "/var/log/apache2/barista-access_log" common
27+
ErrorLog "/var/log/apache2/barista-error.log"
28+
CustomLog "/var/log/apache2/barista-access.log" common
2929
## Proxy.
3030
ProxyPreserveHost On
3131
ProxyRequests Off

‎templates/httpd-vhosts-prod-noctua-ssl.conf

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
<VirtualHost *:443>
2+
ServerAdmin admin@localhost
3+
ServerName {{ noctua_host }}
4+
ServerAlias {{ noctua_host_alias }}
5+
6+
## Setup robots.txt.
7+
DocumentRoot /var/www/html
8+
Alias /robots.txt /var/www/html/robots.txt
9+
RewriteEngine On
10+
RewriteRule ^/robots.txt /robots.txt
11+
12+
## Get aggressive with badly behaving bots.
13+
RewriteCond %{HTTP_USER_AGENT} ^.*Adsbot.*$ [OR]
14+
RewriteCond %{HTTP_USER_AGENT} ^.*AhrefsBot.*$ [OR]
15+
RewriteCond %{HTTP_USER_AGENT} ^.*Amazonbot.*$ [OR]
16+
RewriteCond %{HTTP_USER_AGENT} ^.*Applebot.*$ [OR]
17+
RewriteCond %{HTTP_USER_AGENT} ^.*BingBot.*$ [OR]
18+
RewriteCond %{HTTP_USER_AGENT} ^.*DotBot.*$ [OR]
19+
RewriteCond %{HTTP_USER_AGENT} ^.*Googlebot.*$ [OR]
20+
RewriteCond %{HTTP_USER_AGENT} ^.*infotiger.*$ [OR]
21+
RewriteCond %{HTTP_USER_AGENT} ^.*MauiBot.*$ [OR]
22+
RewriteCond %{HTTP_USER_AGENT} ^.*PetalBot.*$ [OR]
23+
RewriteCond %{HTTP_USER_AGENT} ^.*semrush.*$ [OR]
24+
RewriteCond %{HTTP_USER_AGENT} ^.*WhatWeb.*$ [OR]
25+
RewriteCond %{HTTP_USER_AGENT} ^.*WhatWeb.*$
26+
RewriteRule . - [R=403,L]
27+
28+
29+
ErrorLog "/var/log/apache2/noctua-error.log"
30+
CustomLog "/var/log/apache2/noctua-access.log" common
31+
32+
## Proxy.
33+
ProxyPreserveHost On
34+
ProxyRequests Off
35+
ProxyPass / http://noctua:8910/
36+
ProxyPassReverse / http://noctua:8910/
37+
38+
SSLEngine on
39+
SSLCertificateFile /opt/credentials/fullchain.pem
40+
SSLCertificateKeyFile /opt/credentials/privkey.pem
41+
</VirtualHost>

‎templates/httpd-vhosts-prod-noctua.conf

+2-2
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@
2626
RewriteRule . - [R=403,L]
2727

2828

29-
ErrorLog "/var/log/apache2/noctua-error_log"
30-
CustomLog "/var/log/apache2/noctua-access_log" common
29+
ErrorLog "/var/log/apache2/noctua-error.log"
30+
CustomLog "/var/log/apache2/noctua-access.log" common
3131

3232
## Proxy.
3333
ProxyPreserveHost On

0 commit comments

Comments
 (0)