diff --git a/lib/utils/request.js b/lib/utils/request.js index 92a0e1e9..afaae286 100644 --- a/lib/utils/request.js +++ b/lib/utils/request.js @@ -1,11 +1,17 @@ import requestLib from 'got'; import throttled from './throttle.js'; +import { CookieJar } from 'tough-cookie'; import createDebug from 'debug'; +const cookieJar = new CookieJar(); const debug = createDebug('google-play-scraper'); function doRequest (opts, limit) { let req; + + // cookies are necessary for pagination to work consistently across requests + opts.cookieJar = cookieJar; + if (limit) { req = throttled( requestLib, { diff --git a/package-lock.json b/package-lock.json index 41b0baf9..758aa76a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,8 @@ "debug": "^3.1.0", "got": "^11.8.6", "memoizee": "^0.4.14", - "ramda": "^0.29.0" + "ramda": "^0.29.0", + "tough-cookie": "^4.1.3" }, "devDependencies": { "chai": "^4.3.7", @@ -3224,6 +3225,11 @@ "integrity": "sha512-aEoYC8aTYC/aP8IznQQ1JS6b1wj6P1J73uBrtcyns2IOo3s4lkV4KXUcxK5Hmc5rJnApJCffiI9Z+NrbVTAUCw==", "dev": true }, + "node_modules/psl": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", + "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==" + }, "node_modules/pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -3237,11 +3243,15 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", - "dev": true, "engines": { "node": ">=6" } }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==" + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -3341,6 +3351,11 @@ "node": ">=0.10.0" } }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==" + }, "node_modules/resolve": { "version": "1.22.4", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.4.tgz", @@ -3712,6 +3727,20 @@ "node": ">=8.0" } }, + "node_modules/tough-cookie": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz", + "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==", + "dependencies": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.2.0", + "url-parse": "^1.5.3" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/tsconfig-paths": { "version": "3.14.2", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz", @@ -3842,6 +3871,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/universalify": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", + "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", + "engines": { + "node": ">= 4.0.0" + } + }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", @@ -3851,6 +3888,15 @@ "punycode": "^2.1.0" } }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "node_modules/validator": { "version": "13.11.0", "resolved": "https://registry.npmjs.org/validator/-/validator-13.11.0.tgz", @@ -6360,6 +6406,11 @@ "integrity": "sha512-aEoYC8aTYC/aP8IznQQ1JS6b1wj6P1J73uBrtcyns2IOo3s4lkV4KXUcxK5Hmc5rJnApJCffiI9Z+NrbVTAUCw==", "dev": true }, + "psl": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", + "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==" + }, "pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -6372,8 +6423,12 @@ "punycode": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", - "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", - "dev": true + "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==" + }, + "querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==" }, "queue-microtask": { "version": "1.2.3", @@ -6432,6 +6487,11 @@ "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", "dev": true }, + "requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==" + }, "resolve": { "version": "1.22.4", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.4.tgz", @@ -6690,6 +6750,17 @@ "is-number": "^7.0.0" } }, + "tough-cookie": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz", + "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==", + "requires": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.2.0", + "url-parse": "^1.5.3" + } + }, "tsconfig-paths": { "version": "3.14.2", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz", @@ -6787,6 +6858,11 @@ "which-boxed-primitive": "^1.0.2" } }, + "universalify": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", + "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==" + }, "uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", @@ -6796,6 +6872,15 @@ "punycode": "^2.1.0" } }, + "url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "requires": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "validator": { "version": "13.11.0", "resolved": "https://registry.npmjs.org/validator/-/validator-13.11.0.tgz", diff --git a/package.json b/package.json index 46059f10..62ecf364 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,8 @@ "debug": "^3.1.0", "got": "^11.8.6", "memoizee": "^0.4.14", - "ramda": "^0.29.0" + "ramda": "^0.29.0", + "tough-cookie": "^4.1.3" }, "devDependencies": { "chai": "^4.3.7", diff --git a/test/lib.reviews.js b/test/lib.reviews.js index e32531ef..2091a152 100644 --- a/test/lib.reviews.js +++ b/test/lib.reviews.js @@ -102,4 +102,25 @@ describe('Reviews method', () => { assert.isNotNull(secondPaginationToken); assert.notDeepEqual(data, dataSecondPage); }); + + it('should get same set of reviews on each run', async () => { + const firstPageReviews = await gplay.reviews({ + appId: 'com.facebook.katana', + num: 1500, + sort: constants.sort.HELPFULNESS + }); + const { data } = firstPageReviews; + + assert.equal(data.length, 1500); + + const secondPageReviews = await gplay.reviews({ + appId: 'com.facebook.katana', + num: 1500, + sort: constants.sort.HELPFULNESS + }); + const { data: dataSecondPage } = secondPageReviews; + + assert.equal(dataSecondPage.length, 1500); + assert.deepEqual(data, dataSecondPage); + }); });