-
Notifications
You must be signed in to change notification settings - Fork 0
/
test.sh
executable file
·98 lines (61 loc) · 3.93 KB
/
test.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
#!/bin/bash
#cd "/home/mactan/Drive/sc_ships/SC_Text/orgs/"
#echo -e "<xml> \n" > test.txt
for i in {1..30}
do
#get the thing
#remove leading and trailing quotes
#run through formatter
curl --no-progress-meter 'https://robertsspaceindustries.com/api/orgs/getOrgs?=null' --compressed -X POST \
--header 'User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/114.0' \
--header 'Accept: */*' \
--header 'Accept-Language: en-US,en;q=0.5' \
--header 'Accept-Encoding: gzip, deflate, br' \
--header 'Content-Type: application/json' \
--header 'X-Requested-With: XMLHttpRequest' \
--header 'Origin: https://robertsspaceindustries.com' \
--header 'DNT: 1' \
--header 'Connection: keep-alive' \
--header 'Referer: https://robertsspaceindustries.com/community/orgs/listing?sort=size_desc&search=&pagesize=12&page=1&' \
--header 'Sec-Fetch-Dest: empty' \
--header 'Sec-Fetch-Mode: cors' \
--header 'Sec-Fetch-Site: same-origin' \
--header 'Sec-GPC: 1' \
--header 'TE: trailers' \
--data '{"sort":"size_desc","search":"","commitment":[],"roleplay":[],"size":[],"model":[],"activity":[],"language":[],"recruiting":[],"pagesize":12,"page":'$i'}' \
| jq '.data.html' | sed 's/^.\(.*\).$/\1/' | xargs -0 printf '%b' >> test.txt
#website gets cranky without sleep
sleep 1
done
#echo -e "\n</xml>" >> test.txt
#test="$(cat test.txt)"
#xmllint --html test.txt | sed --regexp-extended 's/&.*;//g' | sed '/^[[:space:]]*$/d' | sed "/'img src'/d" > test2.txt
#test=$(echo $test | sed 's/™/ tm/g; s/ñ/nya/g; s/≡/=/g; s/ó/acutie/g; s/é/ecutie/g; s/©/copywrite/g; s/É/eacute/g; s/í/iacute/g; s/•/bull/g')
#test=$(echo $test | sed --regexp-extended 's/&.*;//g')
#printf "%b " $test | sed 's/\\//g' > test2.txt
filename=`date +"%Y%m%d%H%M".csv`
#get org symbol and member count and format csv
xmllint --html --xpath '*//a//span/span/span[@class="symbol"]/text() | *//span[2]/span[3]/span[2]/text()' test.txt | sed 'N;s/\n/, /;s/$/,/' > $filename
#xmlstarlet sel -t -m "*//a" -v "concat(*//span/span, ', ', *//span[2]/span[3]/span[2], ',')" -n test2.txt > $filename
#xmllint --html --xpath '*//a//span/span/span[@class="symbol"] | *//span[2]/span[3]/span[2]' 1_original.txt > test4.txt
#rm test.txt
#rm test2.txt
head=$(head -n 1 $filename)
tail=$(tail -n 1 $filename)
linecount=$(wc -l < $filename)
message="recorded org stats \ntop org ${head} \nlast org ${tail} \n${linecount} orgs recorded"
notify-send 'Org Stats' "$message" --icon=dialog-information
#python3 -c 'import html,sys; print(html.unescape(sys.stdin.read()), end="")' < file.html
# echo $(xmlstarlet sel -t -v '//span[@class="symbol"]' -v '//span[span = "Members: "]' test2.txt)
# #ORG
# echo $(xmlstarlet sel -t -v '//span[@class="left"]//span[@class="symbol"]' test2.txt)
# echo $(xmlstarlet sel -t -v '*//div/a/span[2]/span[2]/span' test2.txt)
# #COUNT
# echo $(xmlstarlet sel -t -v '//span[@class="right"]//span[span = "Members: "]//span[@class="value"]' test2.txt)
# echo $(xmlstarlet sel -t -v '*//span[2]/span[3]/span[2]' test2.txt)
# #COMBINED
# xmlstarlet sel -t -v '//span[@class="left"]//span[@class="symbol"]' -n -v '//span[@class="right"]//span[span = "Members: "]//span[@class="value"]' test2.txt > test3.txt
# xmlstarlet sel -t -m "*//a" -v "concat(*//div/a/span[2]/span[2]/span, ', ', *//span[2]/span[3]/span[2], ',')" -n test2.txt > test3.txt
# "concat(@id,',',span,',',span)"
# /div[1]/a/span[2]/span[2]/span
# /div[1]/a/span[3]/span[2]/span[3]/span[2]