Merge pull request #72 from bertolo1988/developing

Developing
bertolo1988 · Nov 2, 2016 · 765456b · 765456b
2 parents a15633d + 3de8e92
commit 765456b
Show file tree

Hide file tree

Showing 7 changed files with 59 additions and 61 deletions.
diff --git a/README.md b/README.md
@@ -55,7 +55,7 @@ let defaultOptions = {
     targetDate: Utils.getDefaultDateString(defaultDateFormat),
     database: 'localhost:27017/flight-scrappper',
     collection: 'flight-data',
-    timeout: 50000,
+    timeout: 80000,
     browser: 'chrome',
     chromedriverArgs: [],
     maximize: false,

diff --git a/flight-scrappper-app.js b/flight-scrappper-app.js
@@ -3,15 +3,15 @@ let debug = require('debug')('flight-scrappper-app.js');
 let Utils = require('./src/utils');
 
 let options = {
-    periods: 3,
+    periods: 40,
     interval: 48,
     currency: 'EUR',
     directFlight: false,
     dateFormat: 'DD-MM-YYYY',
     targetDate: Utils.getDefaultDateString('DD-MM-YYYY'),
     database: 'localhost:27017/flight-scrappper',
     collection: 'flight-data',
-    timeout: 50000,
+    timeout: 80000,
     browser: 'chrome',
     chromedriverArgs: ['--verbose', '--log-path=chromedriver.log'],
     maximize: false,

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
     "name": "flight-scrappper",
-    "version": "2.3.1",
+    "version": "2.3.2",
     "description": "Web scraper made with nodejs and selenium-webdriver that gathers flight data and stores it in a mongodb database",
     "homepage": "https://github.com/bertolo1988/flight-scrappper",
     "author": "bertolo1988 <tiagobertolo@gmail.com> (https://github.com/bertolo1988)",
@@ -13,7 +13,7 @@
     ],
     "main": "dist/flight-scrappper.js",
     "engines": {
-        "node": "*"
+        "node": ">= 6.5.0"
     },
     "repository": {
         "type": "git",
@@ -24,14 +24,14 @@
     },
     "license": "MIT",
     "dependencies": {
-        "chromedriver": "2.24.1",
+        "chromedriver": "2.25.1",
         "cross-env": "3.1.3",
         "debug": "2.2.0",
         "file-system": "2.2.1",
         "moment": "2.15.2",
         "mongodb": "2.2.11",
         "progress-barzz": "1.0.2",
-        "selenium-webdriver": "2.53.2"
+        "selenium-webdriver": "3.0.0-beta-3"
     },
     "devDependencies": {
         "jshint": "2.9.4",

diff --git a/src/momondo-scrappper.js b/src/momondo-scrappper.js
@@ -76,16 +76,6 @@ function momondoScrappper() {
         return 'http://www.momondo.co.uk/flightsearch/?' + momondo.toString();
     }
 
-    function takeScreenShot(route, targetDate) {
-        return driver.takeScreenshot().then((data) => {
-            let todayDate = Utils.getTodayDateString('DD-MM-YYYY_HH_mm');
-            let imgName = todayDate + '_' + route.from + '_' + route.to + '_' + targetDate + '.png';
-            let ssPath = 'screenshots' + path.sep;
-            fs.writeFileSync(ssPath + imgName, data, 'base64');
-            debug('Screenshot saved at ' + ssPath + imgName + ' !');
-        });
-    }
-
     function filterSucessfullPromises(promisesMap) {
         var results = [];
         for (let p of promisesMap) {
@@ -134,38 +124,57 @@ function momondoScrappper() {
         });
     }
 
-    function retrieveFlightPage(route, targetDate, dateFormat, currency, directFlight, maximize, timeout) {
-        let fullUrl = buildUrl(route.from, route.to, targetDate.format(dateFormat), currency, directFlight);
+    function resizeWindow(maximize) {
         if (maximize) {
-            driver.manage().window().maximize();
+            return driver.manage().window().maximize();
+        } else {
+            return Promise.resolve();
         }
-        driver.get(fullUrl);
+    }
 
-        let inProgressPromise = driver.wait(() => {
-            return driver.findElement(By.id('searchProgressText')).getText().then((text) => {
-                return text === 'Search complete';
+    function retrieveFlightPage(route, targetDate, dateFormat, currency, directFlight, maximize, timeout) {
+        return resizeWindow(maximize).then(() => {
+            let fullUrl = buildUrl(route.from, route.to, targetDate.format(dateFormat), currency, directFlight);
+            let getPromise = driver.get(fullUrl);
+            return getPromise.then(() => {
+                let inProgressPromise = driver.wait(() => {
+                    return driver.findElement(By.id('searchProgressText')).getText().then((text) => {
+                        return text === 'Search complete';
+                    });
+                }, timeout);
+                return inProgressPromise.then(() => {
+                    return retrieveFlightData(route, targetDate, dateFormat);
+                });
             });
-        }, timeout);
-
-        return inProgressPromise.then(() => {
-            return retrieveFlightData(route, targetDate, dateFormat);
         });
     }
 
-    function handleError(error) {
-        debug(error);
-        return Promise.resolve([]);
+    function takeScreenShot(route, targetDate) {
+        return driver.takeScreenshot().then((data) => {
+            let todayDate = Utils.getTodayDateString('DD-MM-YYYY_HH_mm');
+            let imgName = todayDate + '_' + route.from + '_' + route.to + '_' + targetDate + '.png';
+            let ssPath = 'screenshots' + path.sep;
+            fs.writeFileSync(ssPath + imgName, data, 'base64');
+            debug('Screenshot saved at ' + ssPath + imgName + ' !');
+        });
     }
 
-    function launchRetry(retries, error, route, date, dateFormat, currency, directFlight, maximize, timeout) {
-        if (retries > 0) {
-            retries--;
+    function scrapFlights(route, date, dateFormat, currency, directFlight, maximize, timeout, retries) {
+        return retrieveFlightPage(route, date, dateFormat, currency, directFlight, maximize, timeout).catch((error) => {
+            debug('Caught an error while trying to retrieve the flights');
             debug(error);
-            debug('Retrying...');
-            return retrieveFlightPage(route, date, dateFormat, currency, directFlight, maximize, timeout).catch(handleError);
-        } else {
-            return handleError(error);
-        }
+            return takeScreenShot(route, date, dateFormat).then(() => {
+                debug('Retrying...');
+                return scrapFlights(route, date, dateFormat, currency, directFlight, maximize, timeout, retries - 1);
+            }).catch((err) => {
+                debug('Failed to take screenshot');
+                debug(err);
+                stopBrowser();
+                startBrowser();
+                debug('Retrying...');
+                return scrapFlights(route, date, dateFormat, currency, directFlight, maximize, timeout, retries - 1);
+            });
+        });
     }
 
     function scrap(args) {
@@ -175,26 +184,9 @@ function momondoScrappper() {
         let currency = args.currency;
         let directFlight = args.directFlight;
         let maximize = args.maximize;
-        let timeout = args.timeout || 40000;
+        let timeout = args.timeout || 80000;
         let retries = args.retries || 1;
-        try {
-            return retrieveFlightPage(route, date, dateFormat, currency, directFlight, maximize, timeout).catch((error) => {
-                return takeScreenShot(route, date, dateFormat).then(() => {
-                    return launchRetry(retries, error, route, date, dateFormat, currency, directFlight, maximize, timeout);
-                }).catch((err) => {
-                    debug(err);
-                    debug('Driver crashed');
-                    stopBrowser();
-                    startBrowser();
-                    return launchRetry(retries, error, route, date, dateFormat, currency, directFlight, maximize, timeout);
-                });
-
-            });
-        } catch (error) {
-            return takeScreenShot(route, date).then(() => {
-                return handleError(error);
-            });
-        }
+        return scrapFlights(route, date, dateFormat, currency, directFlight, maximize, timeout, retries);
     }
 
     return {
@@ -204,4 +196,4 @@ function momondoScrappper() {
     };
 }
 
-module.exports = momondoScrappper();
+module.exports = momondoScrappper();
diff --git a/src/options.js b/src/options.js
@@ -11,7 +11,7 @@ function getDefaultOptions() {
         targetDate: Utils.getDefaultDateString(defaultDateFormat),
         database: 'localhost:27017/flight-scrappper',
         collection: 'flight-data',
-        timeout: 50000,
+        timeout: 80000,
         browser: 'chrome',
         chromedriverArgs: [],
         maximize: false,

diff --git a/src/persistency-module.js b/src/persistency-module.js
@@ -27,6 +27,7 @@ function persistencyModule() {
         let insertion = db.collection(collection).insertMany(docs);
         return insertion.then((res) => {
             debug('Persisted ' + res.insertedIds.length + ' results');
+            debug(res.insertedIds);
             db.close();
             return res.insertedIds;
         });

diff --git a/src/utils.js b/src/utils.js
@@ -1,4 +1,5 @@
 var Moment = require('moment');
+const util = require('util');
 var FlightTime = require('../src/flight').FlightTime;
 
 module.exports = {
@@ -12,7 +13,11 @@ module.exports = {
     },
 
     prettifyObject(obj) {
-        return JSON.stringify(obj, null, 4);
+        return util.inspect(obj, {
+            depth: null,
+            colors: true,
+            breakLength: 90
+        });
     },
 
     isNumeric(str) {