-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathbfs.js
53 lines (42 loc) · 2.25 KB
/
bfs.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
'use-strict';
import PuppeteerUtil from "./utils/puppeteerUtil";
import CrawlerUtil from './utils/crawlerUtil';
import HtmlUtil from './utils/htmlUtil';
import logger from './core/logger/app-logger'
export default class Bfs {
static async initilize(node, puppeteer = null, queue, criterion, evaluation, elementsAccessed = [], itens = null, contNodeNumber = 1, withOutSearchKeyWord = false) {
if (puppeteer == null) {
puppeteer = await PuppeteerUtil.createPuppetterInstance();
}
let page = puppeteer.getFirstPage();
const currentPage = page;
try {
const nodeCrawledResult = await CrawlerUtil.crawlerNode(criterion, evaluation, node, page, puppeteer, elementsAccessed, itens, queue, withOutSearchKeyWord);
queue = nodeCrawledResult.queue;
node = nodeCrawledResult.node;
elementsAccessed = nodeCrawledResult.elementsAccessed;
itens = nodeCrawledResult.itens;
} catch (e) {
logger.warn("Click error in criterion" + criterion.name + "in value: " + node.getSource().getValue() + ": " + e);
}
page = currentPage;
while (queue.length > 0 && CrawlerUtil.checkItensComplete(itens) === false) {
logger.info("Criterion: " + criterion.name);
for (let edge of queue) {
console.log("queue nodes: ****:", edge.getSource().value, ' level: ', edge.getLevel());
}
const newNode = queue.shift();
if (newNode.getLevel() > 0 && !HtmlUtil.isUrl(newNode.getSource().getValue())) {
await page.waitForNavigation().catch(e => void e);
await PuppeteerUtil.accessParent(page, newNode.getSourcesParents());
}
return Bfs.initilize(newNode, puppeteer, queue, criterion, evaluation, elementsAccessed, itens, ++contNodeNumber, withOutSearchKeyWord);
}
if (itens === null)
itens = await CrawlerUtil.identificationItens(criterion.name, page, itens, currentPage, evaluation, node);
logger.info("Close Puppeteer ...");
await puppeteer.getBrowser().close()
logger.info("Returnin Criterion: " + criterion.name);
return { "itens": itens, "contNodeNumber": contNodeNumber };
};
}