-
Notifications
You must be signed in to change notification settings - Fork 0
/
config.ini.example.ini
69 lines (61 loc) · 4.81 KB
/
config.ini.example.ini
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
[tor]
ip=127.0.0.1
port=9050
[socks5]
ip=127.0.0.1
port=7890
;[mysql]
;host=127.0.0.1
;port=3307
;user=root
;password=root
;database=darkweb_crawler
;charset=utf8mb4
[mongo]
;host=127.0.0.1
host=xxx
port=27017
username=xxx
password=xxx
database=darkweb_crawler
onion_list_collection_name =onion_list
onion_content_collection_name =onion_content
[setting]
;设置爬取url之前先判断onion_content数据库是否已经爬取过了，先不用了，用下面这个。
whether_crawl_again=1
;设置如果该网站上次爬取时间距离现在超过多久就会重新爬取，否则跳过，单位是天
internal_time_to_crawl=30
time_sleep_to_next_crawl=10
user_agents = [
"Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60",
"Opera/8.0 (Windows NT 5.1; U; en)",
"Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0",
"Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)"]
;基于暗网索引网站/github等网站获取onion域名
[first_websites]
;onion_list=["http://torlinkv7cft5zhegrokjrxj2st4hcimgidaxdmcmdpcrnwfxrr2zxqd.onion","https://github.com/gt0day/DarkWeb"]
;onion_list=["http://juhanurmihxlp77nkq76byazcldy2hlmovfu2epvl5ankdibsot4csyd.onion/add/onionsadded/"]
;onion_list=["https://github.com/gt0day/DarkWeb"]
onion_list=["https://github.com/gt0day/DarkWeb","https://github.com/alecmuffett/real-world-onion-sites","https://github.com/5ky1ar/Awesome-Onion-Links","https://github.com/ajvb/awesome-tor","https://github.com/darknet-book/tor-guide","http://juhanurmihxlp77nkq76byazcldy2hlmovfu2epvl5ankdibsot4csyd.onion/address/","http://torlinkv7cft5zhegrokjrxj2st4hcimgidaxdmcmdpcrnwfxrr2zxqd.onion/","http://zqktlwiuavvvqqt4ybvgvi7tyo4hjl5xgfuvpdf6otjiycgwqbym2qad.onion/wiki/index.php/Main_Page","http://3bbad7fauom4d6sgppalyqddsqbf5u5p56b5k5uk2zxsy3d6ey2jobad.onion/discover","http://juhanurmihxlp77nkq76byazcldy2hlmovfu2epvl5ankdibsot4csyd.onion/add/onionsadded/","http://jaz45aabn5vkemy4jkg4mi4syheisqn2wn2n4fsuitpccdackjwxplad.onion/","http://s4k4ceiapwwgcm3mkb6e4diqecpo7kvdnfr5gg7sph7jjppqkvwwqtyd.onion/","http://darknetlidvrsli6iso7my54rjayjursyw637aypb6qambkoepmyq2yd.onion/onions","http://oniodtu6xudkiblcijrwwkduu2tdle3rav7nlszrjhrxpjtkg4brmgqd.onion/","http://xmh57jrknzkhv6y3ls3ubitzfqnkrwxhopf5aygthi7d6rplyvk3noyd.onion/"]
[third_search]
;keywords=["暗网"]
;50个关键词
keywords=["暗网","摄像头","apk","炒币","Tor","福利","同城","新闻","音乐","电影","加密货币","编程","游戏","VPN","科技","二次元","直播","Telegram","AI","金融","吃瓜","中文","缅北","偷拍","百家乐改赢","小说","抖音","猎奇","素材","社工库","跑分","破解","推特","接码","汉化","漫画","微信","主播","代理","赚钱","区块链","爬虫","索引","数据","色情","onion","forum","drug","hack","erotica"]
[fourth_tor2web]
;28个tor2web的关键词
;keywords=["onion.to"]
keywords=["onion.to","onion.sh","onion.city","onion.direct","onion.top","onion.case","onion.best","onion.guide","onion.run","onion.rip","onion.rent","onion.lu","onion.ru","onion.plus","onion.cafe","noino.run","grams.site","tor2web.io","tor2web.fi","tor2web.org","tor2web.ch","tor2web.blutmagie.de","hiddenservice.net","torstorm.org","tor2storm.org","onion.cab","onion.link","onion.lt"]