-
Notifications
You must be signed in to change notification settings - Fork 0
/
news.js
98 lines (88 loc) · 3.11 KB
/
news.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
const request = require('request');
const MongoClient = require('mongodb').MongoClient;
const apiKey = 'b961f9b8115e429da19aa7f2106f5936';
const keywords = '"deep learning"';
const lastWeekDate = new Date(new Date().setDate(new Date().getDate()-7));
const todayDate = new Date();
const sources = "ars-technica, bloomberg, business-insider, cnbc, engadget, fortune, hacker-news, les-echos, mashable, recode, techcrunch, techradar, the-economist, the-guardian-uk, the-next-web, the-verge, the-wall-street-journal, wired";
// setting options for the fetch request
const options = {
url: 'https://newsapi.org/v2/everything?',
qs: {
'q': keywords,
'sources': sources,
'language': ['en', 'fr'],
'domains': "techcrunch.com, engadget.com, wsj.com",
'from': lastWeekDate,
'to': todayDate,
'pageSize': 100
},
headers: {
'Authorization': 'Bearer ' + apiKey,
'Content-Type': 'application/x-www-form-urlencoded'
}
};
// connection to the DB and insertion of fetched articles
function getArticles () {
MongoClient.connect('mongodb://localhost:27017/watchDB', (err, client) => {
if (err) return console.log(err);
console.log("successfully connected to watchDB");
const db = client.db('watchDB');
storeArticles(db, () => {
console.log("article extracted & stored");
client.close();
});
})
}
// object constructor for mongoDB object
function Article(sourceName, title, description, url, publishedAt) {
this.source = sourceName;
this.title = title;
this.description = description;
this.url = url;
this.date = publishedAt;
}
// fetch article with newsAPI.org and insert them into the DB
function storeArticles (db, callback) {
const search = request(options, (err, res, body) => {
if (err) return console.log("Failed to get sources: " + err);
const data = JSON.parse(body);
console.log("Data obtained from NewsAPI");
const length = data.totalResults;
let i = 0;
while (i < length) {
const collection = db.collection('test4');
let res = data.articles[i];
console.log(res);
console.log(i);
let doc = new Article(res.source.name, res.title,
res.description, res.url, res.publishedAt);
insertDocument (collection, doc, () => {
console.log("Next document");
callback();
});
i++;
}
});
}
// async function for document insertion into the DB
async function insertDocument (collection, doc, callback) {
try {
const count = await collection.find({"url": doc.url}).count();
if (!count) {
const result = collection.insert(doc, (err, result) => {
if (err) return console.log(`Error during insertion: ${err}`);
console.log("document inserted");
callback();
});
} else {
callback();
}
}
catch (err) {
console.log(err);
}
};
module.exports = {
getArticles: getArticles
};