diff --git a/dist/flight-scrappper.js b/dist/flight-scrappper.js index 5d19048..fee64ab 100644 --- a/dist/flight-scrappper.js +++ b/dist/flight-scrappper.js @@ -29,26 +29,13 @@ function flightScrappper() { return Utils.flattenArray(args); } - function launchScrapper(route, date) { - debug('Query: from:' + route.from + ' to:' + route.to + ' date:' + date.format(options.dateFormat)); - return MomondoScrappper.scrap({ - route, - date, - dateFormat: options.dateFormat, - currency: options.currency, - directFlight: options.directFlight, - maximize: options.maximize, - timeout: options.timeout, - retries: options.retries - }); - } - function run(args) { let dates = init(args); let persistPromises = []; for (let route of options.routes) { for (let date of dates) { - let scrapPromise = launchScrapper(route, date); + debug('Query: from:' + route.from + ' to:' + route.to + ' date:' + date.format(options.dateFormat)); + let scrapPromise = MomondoScrappper.scrap(route, date, options.dateFormat, options.currency, options.directFlight, options.maximize, options.timeout, options.retries); persistPromises.push(scrapPromise.then(persistData)); } } diff --git a/package.json b/package.json index e1e6ec6..588dced 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flight-scrappper", - "version": "2.3.2", + "version": "2.3.3", "description": "Web scraper made with nodejs and selenium-webdriver that gathers flight data and stores it in a mongodb database", "homepage": "https://github.com/bertolo1988/flight-scrappper", "author": "bertolo1988 (https://github.com/bertolo1988)", @@ -26,12 +26,12 @@ "dependencies": { "chromedriver": "2.25.1", "cross-env": "3.1.3", - "debug": "2.2.0", + "debug": "2.3.0", "file-system": "2.2.1", "moment": "2.15.2", "mongodb": "2.2.11", "progress-barzz": "1.0.2", - "selenium-webdriver": "3.0.0-beta-3" + "selenium-webdriver": "3.0.0" }, "devDependencies": { "jshint": "2.9.4", diff --git a/src/momondo-scrappper.js b/src/momondo-scrappper.js index a427217..cff641f 100644 --- a/src/momondo-scrappper.js +++ b/src/momondo-scrappper.js @@ -159,36 +159,24 @@ function momondoScrappper() { }); } - function scrapFlights(route, date, dateFormat, currency, directFlight, maximize, timeout, retries) { + function scrap(route, date, dateFormat, currency, directFlight, maximize, timeout, retries) { return retrieveFlightPage(route, date, dateFormat, currency, directFlight, maximize, timeout).catch((error) => { debug('Caught an error while trying to retrieve the flights'); debug(error); return takeScreenShot(route, date, dateFormat).then(() => { debug('Retrying...'); - return scrapFlights(route, date, dateFormat, currency, directFlight, maximize, timeout, retries - 1); + return scrap(route, date, dateFormat, currency, directFlight, maximize, timeout, retries - 1); }).catch((err) => { debug('Failed to take screenshot'); debug(err); stopBrowser(); startBrowser(); debug('Retrying...'); - return scrapFlights(route, date, dateFormat, currency, directFlight, maximize, timeout, retries - 1); + return scrap(route, date, dateFormat, currency, directFlight, maximize, timeout, retries - 1); }); }); } - function scrap(args) { - let route = args.route; - let date = args.date; - let dateFormat = args.dateFormat; - let currency = args.currency; - let directFlight = args.directFlight; - let maximize = args.maximize; - let timeout = args.timeout || 80000; - let retries = args.retries || 1; - return scrapFlights(route, date, dateFormat, currency, directFlight, maximize, timeout, retries); - } - return { scrap, startBrowser, @@ -196,4 +184,4 @@ function momondoScrappper() { }; } -module.exports = momondoScrappper(); \ No newline at end of file +module.exports = momondoScrappper(); diff --git a/test/momondo-test.js b/test/momondo-test.js index f924686..272482a 100644 --- a/test/momondo-test.js +++ b/test/momondo-test.js @@ -79,17 +79,11 @@ describe('momondoScrappper test', function() { }); it('should retrieve [] if there are no flights', () => { - let scrapPromise = MomondoScrappper.scrap({ - route: { - from: 'POR', - to: 'PHI' - }, - date: getDefaultMoment(), - dateFormat: options.dateFormat, - currency: 'EUR', - directFlight: false, - maximize: false - }); + let route = { + from: 'POR', + to: 'PHI' + }; + let scrapPromise = MomondoScrappper.scrap(route, getDefaultMoment(), options.dateFormat, 'EUR', false, false, options.timeout, 1); return scrapPromise.then((flights) => { (flights.length).should.be.exactly(0); }); @@ -109,14 +103,7 @@ describe('momondoScrappper test', function() { let scrapPromise = []; for (let route of routes) { for (let date of dates) { - scrapPromise.push(MomondoScrappper.scrap({ - route, - date, - dateFormat: options.dateFormat, - currency: 'EUR', - directFlight: false, - maximize: false - })); + scrapPromise.push(MomondoScrappper.scrap(route, date, options.dateFormat, 'EUR', false, false, options.timeout, 1)); } } return Promise.all(scrapPromise).then((flights) => { @@ -130,4 +117,4 @@ describe('momondoScrappper test', function() { }); }); -}); \ No newline at end of file +});