Skip to content

Commit

Permalink
Don't store the user data
Browse files Browse the repository at this point in the history 
Currently, some runs create a folder like
`/tmp/puppeteer_dev_chrome_profile-nXlTSz`
with 13Mo of data

See also:
puppeteer/puppeteer#6414

Concretely:
- Use `--incognito`
- Better browser close
- Set `userDataDir` to `/dev/null`
  • Loading branch information
@sbrunner
sbrunner committed Sep 24, 2024
1 parent 11cdca7 commit a93cce7
Showing 1 changed file with 157 additions and 174 deletions.
331 changes: 157 additions & 174 deletions buildtools/check-example.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,13 +61,9 @@ const OSMImage = fileMock('osm.png', 'image/png');
const ASITVDCapabilities = fileMock('asitvd.capabilities.xml', 'text/xml');
const SgxCapabilities = fileMock('sgx.capabilities.xml', 'text/xml');
let browser;
let browserClosed = false;

process.on('unhandledRejection', async (reason, promise) => {
console.log('UnhandledRejection: ', promise, 'reason:', reason);
if (browser && !browserClosed) {
await browser.close();
}
process.exit(2);
});

Expand All @@ -88,7 +84,6 @@ function loaded(page, browser) {
console.log(`The page take more than 60s. to load (${(new Date() - start) / 1000}).`);
console.log('Pending requests:');
requestsURL.forEach((request) => console.log(request));
await browser.close();
process.exit(2);
} else {
timeout = undefined;
Expand All @@ -105,201 +100,189 @@ function loaded(page, browser) {
.then(
async () => {
console.log(`Screenshot saved at: ${screenshotPath}`);
await browser.close();
},
async (e) => {
console.log(`Screenshot error: ${e}`);
await browser.close();
process.exit(2);
},
);
}, 1000);
} else {
await browser.close();
}
}
}, 500);
}
(async () => {
browser = await puppeteer.launch({
args: ['--no-sandbox', '--disable-web-security', '--single-process'],
headless: true,
});
const page = await browser.newPage();
browser = await puppeteer.launch({
args: ['--no-sandbox', '--disable-web-security', '--single-process', '--incognito'],
headless: true,
// Don't store the user data
userDataDir: '/dev/null',
});
process.on('exit', (code) => {
browser.close();
});
const page = await browser.newPage();

await page.setViewport({width: 900, height: 1080});
await page.setRequestInterception(true);
page.on('pageerror', async (e) => {
console.log('Page error');
console.log(e);
await browser.close();
process.exit(2);
});
page.on('dialog', async (e) => {
console.log('Unexpected alert message');
console.log(e);
await browser.close();
process.exit(2);
});
page.on('request', (request) => {
const originalUrl = request.url();
if (process.env.CI != 'true') {
loaded(page, browser);
}
let url = originalUrl;
if (process.env.CI != 'true') {
if (process.env.HTTP_MAP) {
const http_map = JSON.parse(process.env.HTTP_MAP);
for (const key in http_map) {
if (url.startsWith(key)) {
url = url.replace(key, http_map[key]);
}
await page.setViewport({width: 900, height: 1080});
await page.setRequestInterception(true);
page.on('pageerror', async (e) => {
console.log('Page error');
console.log(e);
process.exit(2);
});
page.on('dialog', async (e) => {
console.log('Unexpected alert message');
console.log(e);
process.exit(2);
});
page.on('request', (request) => {
const originalUrl = request.url();
if (process.env.CI != 'true') {
loaded(page, browser);
}
let url = originalUrl;
if (process.env.CI != 'true') {
if (process.env.HTTP_MAP) {
const http_map = JSON.parse(process.env.HTTP_MAP);
for (const key in http_map) {
if (url.startsWith(key)) {
url = url.replace(key, http_map[key]);
}
}
request.continue({
url,
});
return;
}
if (url.startsWith('http://localhost:8080/')) {
url = url.replace('http://localhost:8080/', 'https://geomapfish-demo-2-9.camptocamp.com/');
}
if (url == 'https://ows.asitvd.ch/wmts/1.0.0/WMTSCapabilities.xml') {
request.respond(ASITVDCapabilities);
} else if (
/^https:\/\/[0-9-]*(-test)?\.geomapfish-demo\.[0-9a-z-.]*\.camptocamp\.com\/static-geomapfish\/[0-9a-f]*\/locales\/en.json/.test(
url,
)
) {
request.respond('{}');
} else if (url.startsWith('https://cdn-icons-png.flaticon.com/512/3428/3428903.png')) {
request.continue({
url,
});
return;
}
if (url.startsWith('http://localhost:8080/')) {
url = url.replace('http://localhost:8080/', 'https://geomapfish-demo-2-9.camptocamp.com/');
}
if (url == 'https://ows.asitvd.ch/wmts/1.0.0/WMTSCapabilities.xml') {
request.respond(ASITVDCapabilities);
} else if (
/^https:\/\/[0-9-]*(-test)?\.geomapfish-demo\.[0-9a-z-.]*\.camptocamp\.com\/static-geomapfish\/[0-9a-f]*\/locales\/en.json/.test(
url,
)
) {
request.respond('{}');
} else if (url.startsWith('https://cdn-icons-png.flaticon.com/512/3428/3428903.png')) {
request.respond(OSMImage);
} else if (url == 'https://sgx.geodatenzentrum.de/wmts_basemapde/1.0.0/WMTSCapabilities.xml') {
request.respond(SgxCapabilities);
} else if (
parse(url).host == parse(page_url).host ||
url.startsWith('http://localhost:') ||
url.startsWith('https://geomapfish-demo') ||
url.startsWith('https://wmts.geo.admin.ch/') ||
url.startsWith('https://wms.geo.admin.ch/')
) {
if (originalUrl.includes('/tiles/') && originalUrl.endsWith('.png')) {
request.respond(OSMImage);
} else if (url == 'https://sgx.geodatenzentrum.de/wmts_basemapde/1.0.0/WMTSCapabilities.xml') {
request.respond(SgxCapabilities);
} else if (
parse(url).host == parse(page_url).host ||
url.startsWith('http://localhost:') ||
url.startsWith('https://geomapfish-demo') ||
url.startsWith('https://wmts.geo.admin.ch/') ||
url.startsWith('https://wms.geo.admin.ch/')
) {
if (originalUrl.includes('/tiles/') && originalUrl.endsWith('.png')) {
} else {
console.log(originalUrl);
requestsURL.add(originalUrl);
if (url.startsWith('https://geomapfish-demo')) {
request.headers().origin = 'http://localhost:3001';
}
console.log(`Request: ${url}`);
if (url.startsWith('http://localhost:') && url.endsWith('/favicon.ico')) {
request.respond(OSMImage);
} else {
console.log(originalUrl);
requestsURL.add(originalUrl);
if (url.startsWith('https://geomapfish-demo')) {
request.headers().origin = 'http://localhost:3001';
}
console.log(`Request: ${url}`);
if (url.startsWith('http://localhost:') && url.endsWith('/favicon.ico')) {
request.respond(OSMImage);
} else {
request.continue({
url,
headers: {
// Don't be intranet
'Forwarded': 'for=8.8.8.8;proto=https',
'Cache-Control': 'no-cache',
},
});
}
request.continue({
url,
headers: {
// Don't be intranet
'Forwarded': 'for=8.8.8.8;proto=https',
'Cache-Control': 'no-cache',
},
});
}
} else if (
url.includes('tile.openstreetmap.org') ||
url.startsWith('https://tiles.openseamap.org/') ||
url.startsWith('https://wms.geo.admin.ch/') ||
url.startsWith('https://ows.asitvd.ch/') ||
url.startsWith('https://ows1.asitvd.ch/') ||
url.startsWith('https://ows2.asitvd.ch/') ||
url.startsWith('https://ows3.asitvd.ch/') ||
url.startsWith('https://ows4.asitvd.ch/')
) {
request.respond(OSMImage);
} else {
console.log(`Abort request on '${url}'`);
request.abort('failed');
}
});
page.on('requestfinished', async (request) => {
const ci = process.env.CI == 'true';
const url = request.url();
requestsURL.delete(url);
if (process.env.CI != 'true' || !url.includes('/tiles/')) {
loaded(page, browser);
} else if (
url.includes('tile.openstreetmap.org') ||

Check failure

Code scanning / CodeQL

Incomplete URL substring sanitization High

'
tile.openstreetmap.org
Loading
' can be anywhere in the URL, and arbitrary hosts may come before or after it.
url.startsWith('https://tiles.openseamap.org/') ||
url.startsWith('https://wms.geo.admin.ch/') ||
url.startsWith('https://ows.asitvd.ch/') ||
url.startsWith('https://ows1.asitvd.ch/') ||
url.startsWith('https://ows2.asitvd.ch/') ||
url.startsWith('https://ows3.asitvd.ch/') ||
url.startsWith('https://ows4.asitvd.ch/')
) {
request.respond(OSMImage);
} else {
console.log(`Abort request on '${url}'`);
request.abort('failed');
}
});
page.on('requestfinished', async (request) => {
const ci = process.env.CI == 'true';
const url = request.url();
requestsURL.delete(url);
if (process.env.CI != 'true' || !url.includes('/tiles/')) {
loaded(page, browser);
}
if (
ci &&
url.startsWith('https://geomapfish-demo') &&
request.headers()['sec-fetch-mode'] == 'cors' &&
request.response().headers()['access-control-allow-origin'] == undefined
) {
console.log(`CORS error on: ${url}`);
console.log('= Request headers');
for (const n in request.headers()) {
console.log(`${n}: ${request.headers()[n]}`);
}
if (
ci &&
url.startsWith('https://geomapfish-demo') &&
request.headers()['sec-fetch-mode'] == 'cors' &&
request.response().headers()['access-control-allow-origin'] == undefined
) {
console.log(`CORS error on: ${url}`);
console.log('= Request headers');
for (const n in request.headers()) {
console.log(`${n}: ${request.headers()[n]}`);
}
console.log('= Response headers');
const response = request.response();
for (const n in response.headers()) {
console.log(`${n}: ${response.headers()[n]}`);
}
if (response.headers()['content-type'] == 'text/html') {
const text = await response.text();
console.log('= Response body');
console.log(text);
}
console.log('= Response headers');
const response = request.response();
for (const n in response.headers()) {
console.log(`${n}: ${response.headers()[n]}`);
}
});
page.on('requestfailed', async (request) => {
const url = request.url();
if (
!url.startsWith('https://www.camptocamp.com/') &&
!url.startsWith('https://o330647.ingest.sentry.io/') &&
!url.startsWith('https://maps.googleapis.com/')
) {
console.log(`Request failed on: ${url}`);
await browser.close();
process.exit(2);
if (response.headers()['content-type'] == 'text/html') {
const text = await response.text();
console.log('= Response body');
console.log(text);
}
loaded(page, browser);
});
page.on('console', async (message) => {
const type = message.type();
const location = message.location();
}
});
page.on('requestfailed', async (request) => {
const url = request.url();
if (
!url.startsWith('https://www.camptocamp.com/') &&
!url.startsWith('https://o330647.ingest.sentry.io/') &&
!url.startsWith('https://maps.googleapis.com/')
) {
console.log(`Request failed on: ${url}`);
process.exit(2);
}
loaded(page, browser);
});
page.on('console', async (message) => {
const type = message.type();
const location = message.location();
if (
!location.url.startsWith('http://localhost:3001/.build/examples-hosted/dist/vendor.js') &&
location.url.startsWith('http://localhost:3001/')
) {
console.log(`Console ${type}`);
console.log(`On: ${location.url} ${location.lineNumber}:${location.columnNumber}.`);
console.log(message.text());
if (
!location.url.startsWith('http://localhost:3001/.build/examples-hosted/dist/vendor.js') &&
location.url.startsWith('http://localhost:3001/')
!message.text().includes(' GPU ') &&
!message.text().includes('CORS') &&
!message.text().includes('Driver Message') &&
!message.text().includes('JSHandle@error') &&
!message.text().includes('Password field is not contained in a form') &&
!message.text().includes('Lit is in dev mode. Not recommended for production!') &&
!message
.text()
.includes('Multiple versions of Lit loaded. Loading multiple versions is not recommended.')
) {
console.log(`Console ${type}`);
console.log(`On: ${location.url} ${location.lineNumber}:${location.columnNumber}.`);
console.log(message.text());
if (
!message.text().includes(' GPU ') &&
!message.text().includes('CORS') &&
!message.text().includes('Driver Message') &&
!message.text().includes('JSHandle@error') &&
!message.text().includes('Password field is not contained in a form') &&
!message.text().includes('Lit is in dev mode. Not recommended for production!') &&
!message
.text()
.includes('Multiple versions of Lit loaded. Loading multiple versions is not recommended.')
) {
await browser.close();
process.exit(2);
}
process.exit(2);
}
});
await page.goto(page_url).catch(async (error) => {
console.log(`Page load error: ${error}.`);
await browser.close();
process.exit(2);
});
loaded(page, browser);
})().catch(async (error) => {
console.log(`Unexpected error: ${error}.`);
if (browser && !browserClosed) {
await browser.close();
}
});
await page.goto(page_url).catch(async (error) => {
console.log(`Page load error: ${error}.`);
process.exit(2);
});
loaded(page, browser);

0 comments on commit a93cce7

Please sign in to comment.