Skip to content

Commit f7ce616

Browse files
RexKramer1RexKramer1
committed
added option update-db to run-script
1 parent ea63e3d commit f7ce616

File tree

4 files changed

+272
-37
lines changed

4 files changed

+272
-37
lines changed

‎assets/scripts/loadAircraftData.sh

+180
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
#!/bin/bash
2+
3+
# Define path to directory with databases
4+
# e.g. '/var/lib/postgres'
5+
pathToDirectoryWithCsv="/var/lib/postgresql/dbContent"
6+
7+
echo -----------------------------------------
8+
echo load aircraft data from opensky-network
9+
echo filtering out duplicate records
10+
echo filtering out duplicate hex codes
11+
echo in upper case notation
12+
echo -----------------------------------------
13+
14+
echo -----------------------------------------
15+
echo import csv-file to temp-file 1 ...
16+
echo -----------------------------------------
17+
18+
psql -c "DROP TABLE IF EXISTS TMP_AIRCRAFT_DATA;" -U beluga -d belugaDb
19+
20+
psql -c "CREATE TABLE TMP_AIRCRAFT_DATA AS
21+
SELECT * FROM AIRCRAFT_DATA WITH NO DATA;" -U beluga -d belugaDb
22+
23+
psql -c "COPY TMP_AIRCRAFT_DATA (HEX,
24+
REGISTRATION,
25+
MANUFACTURER_ICAO,
26+
MANUFACTURER_NAME,
27+
MODEL,
28+
TYPECODE,
29+
SERIAL_NUMBER,
30+
LINE_NUMBER,
31+
ICAO_AIRCRAFT_TYPE,
32+
OPERATOR_NAME,
33+
OPERATOR_CALLSIGN,
34+
OPERATOR_ICAO,
35+
OPERATOR_IATA,
36+
OWNER_NAME,
37+
TEST_REG,
38+
REGISTERED,
39+
REG_UNTIL,
40+
STATUS,
41+
BUILT,
42+
FIRST_FLIGHT_DATE,
43+
SEAT_CONFIGURATION,
44+
ENGINES,
45+
MODES,
46+
ADSB,
47+
ACARS,
48+
NOTES,
49+
CATEGORY_DESCRIPTION)
50+
FROM '$pathToDirectoryWithCsv/aircraftDatabase.csv' WITH
51+
DELIMITER ',' CSV HEADER;" -U beluga -d belugaDb
52+
53+
psql -c "\echo number of imported records in temp-file 1:" -U beluga -d belugaDb
54+
psql -c "SELECT COUNT(*) FROM TMP_AIRCRAFT_DATA;" -U beluga -d belugaDb
55+
56+
echo ------------------------------------------------------------
57+
echo copy to temp-file 2 with additional column hex_low
58+
echo ------------------------------------------------------------
59+
60+
psql -c "DROP TABLE IF EXISTS TMP_AIRCRAFT_DATA2;" -U beluga -d belugaDb
61+
62+
psql -c "CREATE TABLE TMP_AIRCRAFT_DATA2 (HEX CHARACTER VARYING(255) NOT NULL,
63+
HEX_LOW CHARACTER VARYING(255) NOT NULL,
64+
ACARS CHARACTER VARYING(255),
65+
ADSB CHARACTER VARYING(255),
66+
BUILT CHARACTER VARYING(255),
67+
CATEGORY_DESCRIPTION CHARACTER VARYING(255),
68+
ENGINES CHARACTER VARYING(255),
69+
FIRST_FLIGHT_DATE CHARACTER VARYING(255),
70+
ICAO_AIRCRAFT_TYPE CHARACTER VARYING(255),
71+
LINE_NUMBER CHARACTER VARYING(255),
72+
MANUFACTURER_ICAO CHARACTER VARYING(255),
73+
MANUFACTURER_NAME CHARACTER VARYING(255),
74+
MODEL CHARACTER VARYING(255),
75+
MODES CHARACTER VARYING(255),
76+
NOTES CHARACTER VARYING(255),
77+
OPERATOR_CALLSIGN CHARACTER VARYING(255),
78+
OPERATOR_IATA CHARACTER VARYING(255),
79+
OPERATOR_ICAO CHARACTER VARYING(255),
80+
OPERATOR_NAME CHARACTER VARYING(255),
81+
OWNER_NAME CHARACTER VARYING(255),
82+
REG_UNTIL CHARACTER VARYING(255),
83+
REGISTERED CHARACTER VARYING(255),
84+
REGISTRATION CHARACTER VARYING(255),
85+
SEAT_CONFIGURATION CHARACTER VARYING(255),
86+
SERIAL_NUMBER CHARACTER VARYING(255),
87+
STATUS CHARACTER VARYING(255),
88+
TEST_REG CHARACTER VARYING(255),
89+
TYPECODE CHARACTER VARYING(255));" -U beluga -d belugaDb
90+
91+
psql -c "INSERT INTO TMP_AIRCRAFT_DATA2
92+
SELECT HEX,
93+
lower(HEX),
94+
ACARS,
95+
ADSB,
96+
BUILT,
97+
CATEGORY_DESCRIPTION,
98+
ENGINES,
99+
FIRST_FLIGHT_DATE,
100+
ICAO_AIRCRAFT_TYPE,
101+
LINE_NUMBER,
102+
MANUFACTURER_ICAO,
103+
MANUFACTURER_NAME,
104+
MODEL,
105+
MODES,
106+
NOTES,
107+
OPERATOR_CALLSIGN,
108+
OPERATOR_IATA,
109+
OPERATOR_ICAO,
110+
OPERATOR_NAME,
111+
OWNER_NAME,
112+
REG_UNTIL,
113+
REGISTERED,
114+
REGISTRATION,
115+
SEAT_CONFIGURATION,
116+
SERIAL_NUMBER,
117+
STATUS,
118+
TEST_REG,
119+
TYPECODE
120+
FROM TMP_AIRCRAFT_DATA;" -U beluga -d belugaDb
121+
122+
psql -c "\echo number of imported records in temp-file 2:" -U beluga -d belugaDb
123+
124+
psql -c "SELECT COUNT(*) FROM TMP_AIRCRAFT_DATA2;" -U beluga -d belugaDb
125+
126+
echo ------------------------------------------------------------
127+
echo copy to destination table aircraft_data
128+
echo with distinct on hex_low
129+
echo combined with order by hex_low, hex DESC
130+
echo result: for all duplicates we get the first record
131+
echo with original lower case hex code
132+
echo ------------------------------------------------------------
133+
psql -c "TRUNCATE AIRCRAFT_DATA;" -U beluga -d belugaDb
134+
135+
psql -c "INSERT INTO AIRCRAFT_DATA
136+
SELECT DISTINCT ON (HEX_LOW)
137+
HEX_LOW,
138+
ACARS,
139+
ADSB,
140+
BUILT,
141+
CATEGORY_DESCRIPTION,
142+
ENGINES,
143+
FIRST_FLIGHT_DATE,
144+
ICAO_AIRCRAFT_TYPE,
145+
LINE_NUMBER,
146+
MANUFACTURER_ICAO,
147+
MANUFACTURER_NAME,
148+
MODEL,
149+
MODES,
150+
NOTES,
151+
OPERATOR_CALLSIGN,
152+
OPERATOR_IATA,
153+
OPERATOR_ICAO,
154+
OPERATOR_NAME,
155+
OWNER_NAME,
156+
REG_UNTIL,
157+
REGISTERED,
158+
REGISTRATION,
159+
SEAT_CONFIGURATION,
160+
SERIAL_NUMBER,
161+
STATUS,
162+
TEST_REG,
163+
TYPECODE
164+
FROM TMP_AIRCRAFT_DATA2
165+
ORDER BY HEX_LOW, HEX DESC;" -U beluga -d belugaDb
166+
167+
psql -c "SELECT COUNT(*) FROM AIRCRAFT_DATA;" -U beluga -d belugaDb
168+
169+
psql -c "\echo number of distinct records in table aircraft_data:" -U beluga -d belugaDb
170+
171+
172+
echo -----------------------------------------
173+
echo cleanup ... drop temp tables
174+
echo -----------------------------------------
175+
psql -c "DROP TABLE IF EXISTS TMP_AIRCRAFT_DATA;" -U beluga -d belugaDb
176+
psql -c "DROP TABLE IF EXISTS TMP_AIRCRAFT_DATA2;" -U beluga -d belugaDb
177+
178+
echo -----------------------------------------
179+
echo Done. Yippie!
180+
echo -----------------------------------------

‎assets/scripts/loadBelugaDb.sh

-35
Original file line numberDiff line numberDiff line change
@@ -4,41 +4,6 @@
44
# e.g. '/var/lib/postgres'
55
pathToDirectoryWithCsv="/var/lib/postgresql/dbContent"
66

7-
echo -----------------------------------------
8-
echo loading table aircraft_data ...
9-
echo -----------------------------------------
10-
psql -c "TRUNCATE TABLE aircraft_data;" -U beluga -d belugaDb
11-
psql -c "COPY aircraft_data (
12-
hex,
13-
registration,
14-
manufacturer_icao,
15-
manufacturer_name,
16-
model,
17-
typecode,
18-
serial_number,
19-
line_number,
20-
icao_aircraft_type,
21-
operator_name,
22-
operator_callsign,
23-
operator_icao,
24-
operator_iata,
25-
owner_name,
26-
test_reg,
27-
registered,
28-
reg_until,
29-
status,
30-
built,
31-
first_flight_date,
32-
seat_configuration,
33-
engines,
34-
modes,
35-
adsb,
36-
acars,
37-
notes,
38-
category_description)
39-
FROM '$pathToDirectoryWithCsv/aircraftDatabase.csv' WITH DELIMITER ',' CSV HEADER;" -U beluga -d belugaDb
40-
echo Done.
41-
427
echo -----------------------------------------
438
echo loading table airport_Data ...
449
echo -----------------------------------------

‎lib.sh

+90-2
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,22 @@ container_name_server=server
1010
path_postgresql=/var/lib/postgresql
1111
path_db_content=$path_postgresql/dbContent
1212

13+
aircraft_database_zipfilename=aircraftDatabase.zip
1314
aircraft_database_filename=aircraftDatabase.csv
1415
airport_database_filename=airports.csv
15-
aircraft_database_url=https://opensky-network.org/datasets/metadata/$aircraft_database_filename
16+
aircraft_database_url=https://opensky-network.org/datasets/metadata/$aircraft_database_zipfilename
1617
airport_database_url=https://davidmegginson.github.io/ourairports-data/$airport_database_filename
1718

1819
load_beluga_db=loadBelugaDb
1920
load_beluga_db_filename=$load_beluga_db.sh
2021
path_load_beluga_db=assets/scripts/$load_beluga_db_filename
2122
load_beluga_db_output_file=$load_beluga_db-output.txt
2223

24+
load_aircraftdata=loadAircraftData
25+
load_aircraftdata_filename=$load_aircraftdata.sh
26+
path_load_aircraftdata=assets/scripts/$load_aircraftdata_filename
27+
load_aircraftdata_output_file=$load_aircraftdata-output.txt
28+
2329
_ask_user_with_message() {
2430
read -p "$1" choice
2531
case "$choice" in
@@ -35,6 +41,17 @@ _ask_user_with_message() {
3541
esac
3642
}
3743

44+
_ask_user_for_decision() {
45+
read -p "$1" choice
46+
case "$choice" in
47+
y | Y) echo "-> Okay, let's do it ..." ;;
48+
n | N)
49+
echo "-> Okay, we skip that step." ;;
50+
*)
51+
echo "-> Invalid answer." ;;
52+
esac
53+
}
54+
3855
_docker_run() {
3956
echo "Run the containers ..."
4057
docker compose up
@@ -146,7 +163,8 @@ _copy_db_content_to_container() {
146163

147164
_download_aircraft_database() {
148165
echo "Download $aircraft_database_filename from Opensky-Network ..."
149-
docker exec -ti $container_name_db bash -c "wget $aircraft_database_url -O $aircraft_database_filename"
166+
docker exec -ti $container_name_db bash -c "wget $aircraft_database_url -O $aircraft_database_zipfilename"
167+
docker exec -ti $container_name_db bash -c "unzip $aircraft_database_zipfilename -o -j"
150168
echo "-> Download $aircraft_database_filename from Opensky-Network. Done."
151169

152170
echo "Copy $aircraft_database_filename to $path_db_content ..."
@@ -170,12 +188,24 @@ _copy_load_db_script_to_container() {
170188
echo "-> Copy $load_beluga_db_filename to container. Done."
171189
}
172190

191+
_copy_load_aircraftdata_script_to_container() {
192+
echo "Copy $load_aircraftdata_filename to container ..."
193+
docker cp $path_load_aircraftdata $container_name_db:$load_aircraftdata_filename
194+
echo "-> Copy $load_aircraftdata_filename to container. Done."
195+
}
196+
173197
_exec_load_db_script() {
174198
echo "Execute $load_beluga_db_filename on container to populate database with content ..."
175199
docker exec $container_name_db bash -c ". $load_beluga_db_filename" >$load_beluga_db_output_file
176200
echo "-> Execute $load_beluga_db_filename on container to populate database with content. Done."
177201
}
178202

203+
_exec_load_aircraftdata_script() {
204+
echo "Execute $load_aircraftdata_filename on container to populate database with aircraftdata ..."
205+
docker exec $container_name_db bash -c ". $load_aircraftdata_filename" >$load_aircraftdata_output_file
206+
echo "-> Execute $load_aircraftdata_filename on container to populate database with aircraftdata. Done."
207+
}
208+
179209
_load_db_content() {
180210
echo "Load csv files into postgres database ..."
181211
if _check_tables_exist -eq 0; then
@@ -202,8 +232,66 @@ _load_db_content() {
202232
echo "-> File $airport_database_filename already exists. Done."
203233
fi
204234

235+
_copy_load_aircraftdata_script_to_container
236+
_copy_load_db_script_to_container
237+
238+
_exec_load_aircraftdata_script
239+
_exec_load_db_script
240+
}
241+
242+
_update_db_content() {
243+
echo "update ... ask user for download of current version ..."
244+
245+
echo "Download $aircraft_database_filename ... "
246+
247+
if [[ -z $(docker exec -ti $container_name_db bash -c "if test -f $aircraft_database_filename; then echo exists; fi") ]]; then
248+
echo "-> file $aircraft_database_filename does not exist, download required."
249+
_download_aircraft_database
250+
else
251+
if [[ $# -eq 0 ]]; then
252+
_ask_user_for_decision "Do you want to download current version of $aircraft_database_filename (y/n)?"
253+
if [ "$choice" != "${choice#[Yy]}" ] ; then
254+
echo "$aircraft_database_filename exists. Download for update requested."
255+
_download_aircraft_database
256+
else
257+
if [ "$choice" != "${choice#[Nn]}" ] ;then
258+
echo "-> $aircraft_database_filename exists. Download for update not requested."
259+
else
260+
echo "-> Invalid answer: $choice. Operation cancelled. Try again."
261+
exit
262+
fi
263+
fi
264+
fi
265+
fi
266+
267+
echo "Download $airport_database_filename ... "
268+
269+
if [[ -z $(docker exec -ti $container_name_db bash -c "if test -f $airport_database_filename; then echo exists; fi") ]]; then
270+
echo "-> file $aircraft_database_filename does not exist, download required."
271+
_download_airport_database
272+
else
273+
if [[ $# -eq 0 ]]; then
274+
_ask_user_for_decision "Do you want to download current version of $airport_database_filename (y/n)?"
275+
if [ "$choice" != "${choice#[Yy]}" ] ; then
276+
echo "$airport_database_filename exists. Download for update requested."
277+
_download_airport_database
278+
else
279+
if [ "$choice" != "${choice#[Nn]}" ] ;then
280+
echo "-> $airport_database_filename exists. Download for update not requested."
281+
else
282+
echo "-> Invalid answer: $choice. Operation cancelled. Try again."
283+
exit
284+
fi
285+
fi
286+
fi
287+
fi
288+
289+
echo "update ... Loading csv files into postgres database ..."
290+
291+
_copy_load_aircraftdata_script_to_container
205292
_copy_load_db_script_to_container
206293

294+
_exec_load_aircraftdata_script
207295
_exec_load_db_script
208296
}
209297

‎run.sh

+2
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ commands:
2222
docker-rm-image <image_name> Remove image (all when empty or specific container)
2323
docker-rm Remove all containers and images of Beluga Project
2424
load-db Fill database in postgres container if tables were created
25+
update-db update database in postgres container (db maintenance)
2526
tables-exist Check if tables in postgres container were created by spring
2627
env Show current environment variables in .env file
2728
install Install project (build docker images and populate database)
@@ -40,6 +41,7 @@ case ${CMD} in
4041
docker-rm-image) _docker_rm_image $@ ;;
4142
docker-rm) _docker_rm_project ;;
4243
load-db) _load_db_content ;;
44+
update-db) _update_db_content ;;
4345
tables-exist) _check_tables_exist ;;
4446
env) _env ;;
4547
install) _install ;;

0 commit comments

Comments
 (0)