-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathscraper.py
102 lines (74 loc) · 2.85 KB
/
scraper.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
from bs4 import BeautifulSoup as BS
import requests
import json
from pprint import pprint
def check_for_negative(element_style, number):
if "down-color" in element_style:
return -abs(number)
else:
return number
def get_page_source_code(url):
header = {
"user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.152 Safari/537.3"}
response = requests.get(url, headers=header)
soup = BS(response.content, "html.parser")
return soup
def convert_script_tag_to_json(script_tag):
data = str(script_tag).replace(
"</script>", "").replace('<script id="__NEXT_DATA__" type="application/json">', "")
json_data = json.loads(data)
return json_data
def get_trending_coin_data():
url = "https://coinmarketcap.com/trending-cryptocurrencies/"
soup = get_page_source_code(url)
script_tag = soup.select_one("script#__NEXT_DATA__")
json_data = convert_script_tag_to_json(script_tag)
trending_coins = json_data["props"]["initialState"]["cryptocurrency"]["trendingCoins"]["data"]
trending_coin_data = []
for coin in trending_coins:
name = coin['name']
symbol = coin['symbol']
price = coin['priceChange']['price']
price_change_in_7d = coin['priceChange']['priceChange7d']
price_change_in_24h = coin['priceChange']['priceChange24h']
market_cap = coin['marketCap']
volume_24h = coin['priceChange']['volume24h']
d = {
"name": name,
"symbol": symbol,
"price": price,
"price_change_24h": price_change_in_24h,
"price_change_7d": price_change_in_7d,
"market_cap": market_cap,
"volume_24h": volume_24h
}
print(d)
trending_coin_data.append(d)
return trending_coin_data
def get_top_coins():
url = "https://coinmarketcap.com/"
soup = get_page_source_code(url)
script_tag = soup.select_one("script#__NEXT_DATA__")
json_data = convert_script_tag_to_json(script_tag)
top_coins = json_data["props"]["initialState"]["cryptocurrency"]["listingLatest"]["data"]
top_coin_data = []
for coin in top_coins:
name = coin['name']
symbol = coin['symbol']
price = coin['quote']['USD']['price']
change_in_7d = coin['quote']['USD']['percentChange7d']
change_in_24h = coin['quote']['USD']['percentChange24h']
market_cap = coin['quote']['USD']['marketCap']
volume_24h = coin['quote']['USD']['volume24h']
d = {
"name": name,
"symbol": symbol,
"price": price,
"change_24h": change_in_24h,
"change_7d": change_in_7d,
"market_cap": market_cap,
"volume_24h": volume_24h
}
print(d)
top_coin_data.append(d)
return top_coin_data