-
Notifications
You must be signed in to change notification settings - Fork 0
/
scraper.rb
82 lines (75 loc) · 21.5 KB
/
scraper.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
require "mechanize"
require "json"
require "scraperwiki"
def scrape_page(agent, page)
id = "a71d58ad-4cba-48b6-8dab-f3091fc31cd5"
base64_secure_configuration = "qecA4vFkzwIkBwQfOenAeuwFvpXSrDXtI4RddpsUXk5xHGLpfgmjZyxkVgbd/rP1Z32tFpZeD2YW8yaOAWP/ugb3n6SwooGFS9Q2ilzjoDB1aKdisdrgJuZKJ0A4qDAPxmz9SO/4BWdQJLk21m+v1RrJKa+ABD8GA/WyZxaXET+NzNgTXPlEjahTMYBkXJR76sry68VMIoMDQ4pyOn060HgvWdCtpb3amWWVn8DBLS/oKKVq8Q7WV8yxU5qkf0pV9eOVN+glzKozXNzk849F05yrYEPMc6AALksvS4kuQBkkBaAL8INADZYASXAf/EvmyLYPNw7mAOG+rgbr92vZJd5c5/H4Nx+c5lvI+HDF+HsuS46qu5Zp0IFQPtkLK/A1Sx+o8bCo+d4fXP5YHsZ0/WAmEW9xxgYHjX02tnazs+eVamOfAKkzYfGKMgjbmr0WvSyOE04Kp9cdEMoC/B5oK91d52FsQT+cY2A4veSsgNHT1qSAYE5rjwaxIkAEScPrHKL0+gGPWFI8JoI1EU1mmpFF5AxYWqrFW896mOePG0lpE7/ZN1qSgIlfnoX3LFO3bbVmZ/ybP3677SmGCCiMneoZwDBkWaEhMJiQ3MxyG2WdeXSjmMOY525ocPeDYB4Mk4HHQ1sM0Eh5Ykab/aUtWs0hfixEY60IEd7G+KK0vdBp+qi1oDgbpmQWWFPn7Bw8ChNVzDCrhyci0YanAgcEj9Q/82yzo+bbpeOQjdnynY7SvdgvGWkJ8gFNQTtWSEKQ2SWxYSdU7XJD1j9BucJeMbqDV2voxAAMdABQ1yvLlQFLZyyvui4GGqMAVzJs7bGkT53SXRZES9/LBKU11aqFCQW3UbOwKmQipAMJUJLI17AOToApu2klIM1utXSRMBEOuCAWqkpP5xjm3S45yWQZjoBARnX/soS1zXBXw/nU2H5sP+Jfl65JJzAiYaa70k4wTr9rcBQ2IfuYZYNU7psDNIj0B+VjmLjzU3BE4a7RIqaml+2KW5rSjoJ2fjiYmR05JO0uXQFqpPJS5uLXL/syheVIP/lZ0vPVi0s5OhwlYcfxRc/aAUivWfEdVlUBLaN5SGvagVoAlPcXpmo7WvelM1LJYy5cDFiAMOB2ud+yxPfeR4UMAw65ntN6EMIg5V+8adTurWfeNhQA+nkPUXIdP763LOWgNnJf+rtApBoNrDs4o/uHw0GbPgxBzYlYmemZvWZB1QI7872BSYNeC3Gf6QjaYtNUHDfRS+uHFwVBwhSHCtsDizeK1ZZ8mjFiZMbx8s2ZhGtHfTEc0U5cosMVOaWyCNYDSbNjRo9QOqJaLPYkSigDH8N5neLcJXWu29mq8m5Z/4qroQL7X8NR8VYp6qyU7FMWPa594awKjzVHXocGw9NAVQKP6BRSYC+xev8dOxORqDPrZHoJlzBmHak7CTFR3NntfcIMFuQIryaNI3DwPtOt1FIbgnjpJsUrBtY6C/VV3GRLrxQ3fc3ccUVlZbjnikS+/ZOv6+UWa7RJqd/8FHqplCSBvVWjfLNqks/D3VJ8Sdp5G3a8wz86Bi7G5eoLK3+FcNK0zBVy3f8pK9AyTNhQCzn5YS7I4K9KdSbE/W39pa+9laj9F9sY9nq2cf5hmNT9MX4Vhz9CwqXmXW6tXSoaM3jAvKYoohwFrXmt6dLOEhzhyNXtsd6Mnc/ORoGQfpeQiY4tTV/7paxRVSwttiu+rki15wLUNg3GQO0zaE42+7mhGwQRiJN4HZwBqSn/3dG4pqyUEfxG5WHbClc6mu1A2gS1zVfEkfmPzsvHjFvOYDJx+E0w0r23te2Rz6ypwknTVT1SqRTYCCl94amL55u4DUe40RqWtF4HBC1H+Yf9GVx6SqnKiYfBy07eRoTw/jqjS8jHOdqJIV5z9Ry+0HeOraMSFSbclOh75fBpEs+AFdszRNaF9eqBctSNYNNneOG/8jzKPoS3WUuaYoCFM7pZ+isCzrGo3NDi9mDUlsjwbUQ1JWRqUa3102iEQSSePmg5EUCg96RfOTQdzimAi/+xXJeztCOw0KqF3R05zOdyEARw6Ci4FUi+f/C+p83+cE8YE/lcfbEOYVi3/0swM1nIbFyXvP0kR/Ccmh7tHnHULeXfVAdGx5XCtG6EfT1OxGWX+TNsYidfowjEK9RMATTrYRLCty1pOHcL8T6wmK7t445CKe6xwCMJx2wFpcOOy2H6pvteUWqgXWsfXOuNkMZmN7VMaVys8BWHGgqv+pSNl7LiKg2A12Rv2sqXafdXSmNNIQYq4UkZF6qnF01zM+3wYFZf+F13wvMVx7w+ZZxFL6cDxNaZckwUwYvWdWM5slWaDCLiJY5RaqPk8Wd/Iuf6HuYoKsFpg3yf57VT2seZedZGGDEtG/vvKvQrvnwAf9Tui+AoR3oE+1YGIrtyPlXhOGTzodHp9W4+11r/Cd0mlxiN46gb8z+GuvuFHpDK6H72f6RvJGfxnHskEXspwlzRuDaNmBXUX66Vu9FW+M64CQnB/roO2Lolr6Tpaf4eZ6iQjPAlzLcYn5DS742uR68/ufDUety186BwWXMvMeSrWBxrSmqD32ZgNgjyO6EYfkDHfhCDB/c0Hq9sm9O2+1wjV4CAI7Y5DqM8ezpe28Muk5IWuBs77/ZCuSrkOvM7BDerO0NikkMTYeU/fNcHa8DL+GiFakucYhv3BQKFa9S5k6DGtllxVtAui8tWWTS8hxrletjaib2owkWoGzyNk6YubG5LcIwomNEU74rEiaTcGQY60klMYlcWd5ykckfrqKmwlHQtkvjami3UNqdS45HVBVeGXCnZyzwhqgzokWBCpMsyNOPlTcsnmdQk8gdASUNbsVqdTbpQ/k1rpSHVcJ4opDMJT03FOMqRR7bydn2x+DbUY3UMaw6PKRZmutt6nu5cFZYyMYphqmvE5XE04LHidoHuqy9BuBrbOTLOcQWzDcuSjeh5ubmdvQD9ztbSZkkg3oclxXwbQ/38F88U9Vr23j25lVXu7A1i11mDz5Uu8Whs6tJDbqz1+gxUS2o/YI0wcMkjyJ2v/jSDXKjg7EeTroB3GqRrKOrbN663e+GJ9FIEXpwnjFSnG5mNAwLdtLS0Lwr93I2OKpvcwUjqH14HocaWyjAdPYLLGg+/AygGA2w9zTeIgxdHwK997gQzPjDeKqI0pwjdRpu3JH1ySDtRbmzC9RUGrAmg6sGtbBg5hhvKzPWQdMccX6m/H4MYYSDJBk7HgS5aj/5ph6b8TJfMhnHtQd47xmXZPp2SaK1VJ8nRvOZjw0KW4AET+LwaAceI93Rkh8OcwdQu2gurB2F2Pl1SvWLJMZNIx9/1Gll/AwwGncNKdxXQEqJTV9DlNi+tlpc9X4AgxCtwbtNvbQQSkiWgYnetqyj1yrnAuRswRPzkcZPUhQUWBkjErLxYu2e//9c+WsRb9SwupZ9TbvBYMiFcEc/kVT25JsuNEmKxeS/R6jtnTv+OddRRi09jX3Po/eZiw/uL2L58DJkIeuYllza+oPQ+hlH3XvcHj7j2Qzok3ckSY9H7zSUISvmw5+lMeLdVAWpgMoU5RaCtO1SN9XXLIwUGxA1aST90Ob/m9SBasXhyKX+2KlyBk8fiiDXxYf8FOECpTXq/4gknZpCVMB+Fy7FsNQ4OSmi49z0jROIqo8cPTE3xSO3joRXefR5B1aNFkWiJdOWq/o7N0tLRrR/aV1kHmOyv37PRibYOfVCrI+T+V/acr+16oEdUMQ5OsnM8rNoXzcwNlQvmHwYjCkHx17wyV7L7IvDUoTor6qe8EXzJQzuomGcckf0uHlHVMidmoTvp4bI4SxhokEcHMHjRQdqB8XPSGXEqFBxrrpalbdyvDO+3e3Be4g4gG5Nfpsjivn9U0cUJIbn5toXcZ3DG7fcOP6fG6kHtW6Omen+0Nt7WKVcHhpO+NwXOYA3KK9+uWI+Pc4E2l/A8dpweAjYMHj6qya69rqgRbTs5lG0J3TpuXIxBK4fwwnvHvu1GzVBJa1fmZpVF+z0jyUXVXkZUQRBY+8YOeF2QGkE0AgM3mBlQ4xaNTbRLX1tzrc0iZtM9PwMVEGLiN9XYCxOf8r6mQ5E05sxr/G+/Y6WbvENWESpDgD8trHELjeFkvFkBn3fLe9ZLHtyyOcp2HuciObe9GbFqtKQPcZzNz4orHnkWl0n6xAhi+UyEeeW7vBbdRMIptY/jXc2jhv+vDjtLCiWRIgtWWdmJY7ff3dGVgn09MSDZayVJ+fIGGtzJqiep90jQjS1nKh1rbMedVnsuNNjVHPlM4mM2H+wT9efAUYRILYKwKt98AFaS89w3KiUyxK6kTQKiguiEGimXRb1oyjA+eSber5wjyuG+3Gtu4Y+5CM7aeOwq9paJ2ZP1lRrWv8UIeXOVdBvJdm1jbplBK9LME67AG0/7HSq1aip4UdS8uZie831EbdcjWUoIq2euhG6kMWVuMmTGDS0FWFlgFwUYu2BC+9FpSQIVlyMH8Y0iymVOlwDQoKwelRPCERXjVQaQrAWTilcLMvJH0ezmG8xMS98Z8nwPwepuoLCFcpqznB77yBcLUNb6jk7CDvHGNmZZRhRE60pGGJTgZLdjRcYmNFZ1r7fo0ONWlhfYskeZI+BbmTnPNTz8M317VrTCAHS6+p8vYlsl8cLBGQs+XUdY3jyHdCitsRnnqfgA/I1tNyoSH3XD4VqKzmR1ge7QWTdDjFX87abh5gCvjDyOJtKJDI8ndjR02ESdhno+Dsbnf90saOXAXrTNbSu+BmEcuc3Nf8DXSCxzT15Zafzqulj9I7cALcPawLkUrZvYj2ur1XMj76WT1f4ijnONogeJXTmb/aP34XSb14jD8IOJQWAk445DOZ+KbZPfR/fkfTstEMKZKnQHxopw9QwKtR8bn2FItASrqPfdD+4OHec32ClumBoz7f7xOCNcVtoY8wckn0ZrEnlbp4jKvChsYOA2yFSlgAk6zG0OUVH2pN9ZHuH9MV8gXMY/PGtVDCaDJyKjMK36CcMDwDZIwnigJGVUKOr0GzYmbFu6qv8npdHSQviGNPWP9NfYY21X3WacKLOykqAaWPjdWgIFnF+NF9cUPgCOs0geelrjKQPq7nVqD37Bfl+jvNsdu7CTQqzPSsvBDfFEL6P8NSdZKs4KgtJrkwd0ydXJUvxPB9zpxNMHdlM+LhWXUnyQqd8HYZdEMLWu1m0sTCtGmOTvAG1BqiJnf7jH7TwuYCnFKtvc8hEFHElCZF8SjEtVrb1rZY5kX+tZ8H47myiyrN0TFYNxOrCpx0lpNJ+BbX08aWd1K7SUfr/NmWuhyuTmyW2g+uEXt1DZMD76OGjGGDNF4vrlwNInYCczyNZknjrX0OTbHCnxz4ik65jxvKaAsfbcdSjUrkTp5TkS5JVDMoPJGOEffrXCUHjHrzvSSIilkFTXzlZc1FARGeRO3C9B+Mbk9hQwkkW0YLzoxJevaQsg3WOP9OChWifR9sB9YtSZjWx1BBe3zhBHfe4lmH3yb3yT0uK8ErvW7ziaMchzyvEaOEcauYtpuJB7fBcnDJ1s6V60Zg7x11Bu/QZovNz4JEsgpHvMADe+M1r0bN/I3M290raUxILp2n+GHTovB1IIFYqptz3FJiEDI46jpxBztpo5f3ZLIaD45ceD3aoZJ0usbto7DbL8M5632a0QgsEjmzJHpk469NamORJKeoinlertHcLJonVN+DQoWkojIh6gaU0UFGqBTr8Q791SO1oOOf5fc8u9w5+d7B9tZhWqL9ISGhd64LeNVwMZnguPUufBNS9Quo/UPpmm+jYtjZlBwh9TwWcXa0W9tmDekNq5TGXjPYtEr7GpEiA0HxpIMO2UmeHW+ZCvR3rubx8rPzGZqeXgQAgXGfjtLDwpHTDqRXVRmoyu4ckX8/u2xY3kDbcWvfp9Y9bPsB2qi+qPgJ5VmkAcH/uvm1KTpJtGNuBLESAV9L4l5UosbfQ4+Svbxa6n6xb7Kx9P/UOCn3NOEBDf3YElGvqygUIjJ12FPOWx2bTECo0fiaTXjrn5m4V3kNymgeVwQLVNVPzXcPU8gmGbHihqooj0KcoUt7kuL+fc/G/uQbtbbEm/80/nmqDR37VsEXCtG4LrESFnHu//LdLSaynz6OJeeSd/mgKEqBvOod9kz7irBSxezCt1tJegvYJF/7ycyiKiM0/Mwbs8YioscJif7x9uXYKMbEtW6ZzRhyyR1Tyopp8I5qwM5ceKCX3Yd095DHKxmSOgWIr7LJu+osybWSKQJifnAxoLtWzns3nbWddGjJx+vbMRW/wkeTYV5oaJdJd7R7Oq/NNEIBtxswD/LnXFv4qrliOp8Iq4w2Xe57Q2yFCBPg1HOab7FRUQK+v+o172CWsne2Avxfjhm86Y95yJDijx/J+GVO6vYlpbcS3UblG8bU/5CLzypEZOc52mj136A5xqPfcLtiC9SZeic/pyoEETfsri/VHU2CGSZKgn6aw4TcNiR+bNl43XVcrGQzX6+Mfj49hM8K/3dj4a+Tja69HVV1d2nbJkC+MrP+Zt0YkhQMDBw+nfs+UVla+HB0Z/unD223jpNSSchrRC0rDraUrLWeLz0L2qSEhqPa0n39J4EG3SLaNjYtNM9dNMbDSqYTJut9rN7wB7YEUHEgif2BqRr0dNPfKdOQFlz9a0y2MMQMvQUeDsBbHJC8hGvirJ8+VJDmdp69YjgLAn6mAyW8ruKAbjocDv6E9EiBAaWX2CInLEwHnT6LVLZUZfFCddI4qK14HtirsRI9zjiD/Z4MUtyGEfa96A+0TzObBaiTnrMJndAM4pYLSE0HmQt9HAh6yM6tlDBv9VV5ie4270wHVpRi/6KDeH5qCtuAOFqH7oLL/ggwDNfwBb6omJA8MudGAGbpDLFe6COTZFCYVjshJ9R5mJw2aPPBmgi7Xp4UGviSb1ieGVR/lnsGZH+9YJDK3Zbp1mfY2QAiGC44QCqd/PnE9PiMrlURNhlM4dDY0mO5qVaXh0yfF6jcVOL9DcIV+qxNCukPklwMZ+1sKlbcxLuIqA+LcCDsZwLNDZ5Rn+HeYHOvEEsEu8cOaTFNT+RYrpGoupehS6oLxWFKfWo3syUsIMFOJs2NPvSqSAm1hA5kVv045KHHGpsWUjVumnL3NdMipXtd1ekDUIbvRpORkSFEn/hMeYmQEFFeeTuzwtEatS9C0s/thkiEXiBKFUZrR53GEx8VEfc+h8bol6Uym2aNoQDajumPkuk4t9szGScNo5tktmEQvuxHVeEpHSRYyxGG3/8YfAX3DbD5VeO18u7KRMgQUE2kn+I8CFGYRajuT3c6E+TMj3kRx5QYwq7TJFiPsb2kgosbzVLF6Hy+I+lXluE3O8pAgXSkUUkTSJqrzl1m5ISerMFIpboskEXgWGIPtULXdnnO9PxjhPBvxVUFXHihD/0n8N+PAwGgBpKkJyg2J+6mJC6OWgLv1JnfQgwHThbgxztk6/NsNGQGAYQxVrzhndAy+dvCVZ0FYv80cWvfyFbbU4347W32vJHzzjCmzd3CIrBRW12N4+milOIeEX8ldhV8cwaFXHJ9uHrH+uV5rjW3Phl6npf/DMJnjIsQHIDD9lsvMB2ApxWbTaDxgCLL2tGlbtPUpJecXIeIPHGeynrXR67A5bxWSbGvklKVwia2343dieJ1EdxylgiFg9y5Lpk+8kxeknDD2n3r7DT6nVOANdYzZkq9gxWDTxSdN68cnPoSu/DASdIesbWM/4vUsOHibEdFsXPLg8vkAw+keeuTEYWLdVhF4HFpHmAhp7VkUyyL4upEFjMSTimMP+959QR77ksqt51WLvRKFVkbePRotu8wh28Do1lbNFn5OeBEoxm3yibF5t0676FXtxkLfLb8uyVK56yKehhTjsBW7DLfThbeaqcMRBSsHVmTcPwHtWjbBKvukDTVBWbsfUUSS3yVeHe4iuG6rwTEM5zlxf1uR8PFyBcIDPGlsPpljsVE7jfnhEKjia8PrfkWf8/uHyCvN+aK0Viog7qvMKHaY4OGP2oqoce9VWTqDCX0eQOrkm8fyvvPmy92ugoMotyYOWZzzHMPJZN4kF/Cf+yvgoBqFc2ggn/+udF4nNejLPKqs+96woASvu1nqkwJjcMbsXWsG1SL7gGDJkQ7z5ScOWdk3Vz1fHnxVCHUCgMyZ/VAK0AwOdgUbqroJDxu4DOZR1y9Qfv8U9z9FvKFcTV75h8JNdVDXTTvd8n9CFo1CYhSIoLrsWT0ghQGMhLxHzEq07BNUuddUUmV31YGphMZCc6slugpFDmtC84gdrNYVxals+C15aeUwefKZkeXZcUsCzVC5jjgSA56BKgpKVqtaPd49UQPxOWRdm8X98+XfOE8rbCbdrDqM77VKVG0w+XtBEZCNE7B6NlcXeuoXuZlKadezHAyJdpUcBJ+zqfdRn+EARVDG/UaOlx17NuglRMyFgyckLwyZ7Gcz7eI8OHrztYw1k/u+GVomt65sR7d9AqUiVy0lHRv5l1tE+orQDnZAXLHTJfaum7bf7GLsPE5oMtpcnzJP7CTDBnQxrZBFQaTQaR0cyJUOj2+fH6l2LOKeXwuJuuwwTyFbucTjGIqFucgeJLl4VDLOhEWGZiADauC9dOD0ijv53PlX9+dJ6BwEDYAoLnAsDnnUSlTabImR2aEOPfQkISQJIHATcejk5FCAt5r/HS7QCNkma1Y0J+O4T7XC4KD/tjIOoRq1sk+qlEONEtVsqBHGNyRyha2fiT5medoLeYqymdm20rEIisuAB+0rGrZlLswsL1ZTfTasTRZGsoka7k6NhnrMhZ+f/RPG1V7D6I0CjG/n+DV8rmnYkNxHJbW+VoKJCWTCLceF/TOhuh3UMtTtTRscPs6waQJxPRgqcRczt7F1cHkm6dH1Wij3PMIx9yNLX5LZ5fg81aBUonE8Cdlrh/+7Ff2TsNX1dxdp4/nC1x7crSM0hMY6MmuJzvJdMhLoBmorqHtrijiS10aPGHu+aj1KF7KbSNMLf8jH/4T9SbeHqVyY2gGdZqGAluOK7uFJWeeuOe9SWUbq3Rzvo8PJ2qJBHdlVvEdmz7464cGPPd1WGcwAvTZ8nQlglkJGS9uQ6p37/yYdIZQyihbe3CFHy4mtk4fjTCBg7TdUR/NX93CkMTiyf+afXpm1QgkDQe1UqJtvBHgx/Edyke7CKbORByRvgid/1xJ4rSp0GtSMA3awvRnJSLRcKP4uPGHRVrO1a6/7p6WRcyZYuRfa+lFv4E2k8CN9/BxHE6wlPDJDm4zQuKzrLP0Th7bjNYeN7L6DSTObc5OzfaQOsRX8E1c8GdXtCd1wxxv6Q2f27TQ/2wZMarwlkvnm51JcUnMXL66FuJACf6h1Vf70ZhCFZYz3/KfmxQT5vAi+M+sxIuEYOv9LwO+tMZmsj3XZAVp331s5Qqc22Yxjhq6xb7ElDy8Qt0b0XsOL2hzi6qvw53tRe8smKxR9/NEnG2biT+ofTBoQSmgN0l8S7nJGqT/du6GYzcWFfAFtvTQZ12Dml7e28oGTtHGVNrB8qMsPZTIQUebpbxIhmEY0C8W0nJSfYgQqr3KAo1lamygM6Pcq7uo+1B/EQKIm1TzL/0zXhIgV0jLkgBHArW244ivq4slGqpdXQiNZ5MyFc+284PwBknfVIgk8zx9NoDyzO4iqZlyuCoXsd8TL+PRP0/2rQpPIthHhkXq19Wd+YDAJevFNSoJkhsavCkma8fMI4G9dvH/0LvqfmMd6lKel6VaZ8xrBbalRIGz3wYECX+sU2VRrnaKtihL8OSSmaArQQa6lwlu6aByS+jtkDfpmdr1zBVvUExZnPKlyAjYBZt1ofVI3RuT5XU++XlKCi24nkYwlRcbHlclRPn7i04r4qhqlXRiGtUVq8GnpfnUtx+FtJ99RLvBmNNn6MjBxbXjKbagNnejH8D9w19N3KyMfFlA0uYdWIj45n/VMePcSANFyGCRZfY2w/OXaI8Cs2bPpxtXXiwq4KO1rH0g7Tks6eu0iPaMCiGi/ptVgaOR3adatrJAUAxF1MYhjhcOnVfpw5CVWPrB7gd1nW+L9/v6pemkK3bki2++gqaQUou24YRR9UF7PJCTodQble5Q+0LT42yQ65N05VOKsClBLU2/oGc7Y+jYP8PdnKI/CmRWKXhE28pzYUs9ideG/a20ZMIFP4+2RXp8a7ERNIGWGfX5dz+EGstN3/yBuVOGwd39plZoJh0i0z72W2SUm/xBYsJXjCcm6YBpWVnm5n5BsB5jqRl2nOM5ScGt+IQutJy15cSwu56zeveFBP+AvcFHq927Pp6tSLfA5+tt+aDvoX1k/EWMVoMQaWy67L89AwcOEPlXQMdQLmF2wTFtk0VIEVXzkYFDITMRQ1HVpLYRwzRyJm8nk2/CJcGA8m1rLWFtQoPea7m1h90dmxe4odKIJxDu3ZZzLoLt4MXagYe4q4tdF++wrAJvKz4GhfcxSkFMmxFH4u1VK9qjj9ksgeV/eJf66yy64RW0ZKA+K00WmGSoJvtLjlJVxIefqED3ySeaVVwrFQ/E11d28zI7O2DwpBtsu5PGBUUQfOjDWP3ACQT1+Uuno52ahlODIMXJSZg2klp/Mbap8H+mdkxHh7NkRldzpjqJpNIvqXEUniUZVMdYpLLWQBQMQ+rplkGCAH8PhFm6SQHjyp6P4RQUQcH9xhGIakRHgVNnp7mNnCReHLUYbgdxoeXmqxj+HR5O9NFssUwJ/S4ljeKG+6WQqeyTx2fpK+9xIUyao5VUPDglwV3xmWIgnougmWwyoAlJG7cSqu57cV83xEqRnWePuCOwneIcihUIMOYXCKmmFCqDIzfNOzbzhgW+3oxJjmVvqRye9jx+UEjmIAyep0+ZjZT5IunyUa72bjET71/GkGRCzwV9Ez38CDmP68CpqS6dMDNPDUUfcYuOMR20xYOIuQFWXczC5l8fllRk6gMlS5ugw4QzKuvUwlr92Q3uho5N0cZYIPlu/N/mEBdo9IUP6tPI81wsadAJm4psER19VpIxDrrnvTmPW5vjIPq4f4duBTJuPC6cMt17C/sYdd+Dba1GGHTPs99a/9oesQn33rEeAW6UbHYVabzPkQxZBMtBJQKqWxM55OgCL/UtbT8JN9YbaviCYIXJXoSUw10MbeZRNqv6g4rNXJ4EiXjetEqeYjSdaIyh6yMu2d/GHokLUSC40sO2q6lwt/IB2nKzUU6kZfqQ9VYMSIeriGmuOLrR0X6Mh0Pbs5A1+WOuUD2DMDibwCugZGFx6qWN7n9lfJnGGmcstKodkeGbQcTPHhjY1XIMYGk+lwpz7WFksnkDmlDon7fvUDWQrFkfrRYSw5qVZT+vHJoBdWxOkRtSBNCQ5gasTUl5QWS7481tsOcScgYX0qfjgJCsyvu2Sg0M4+eQg2pVMARwrH5mbBsIHvnQvARKqzwJiNOg1Ebcv7WKcXcxj07tMdsjq5demDhvD7OiKf70lah/h3G0luBaXs/WIOQV72Ncv3BfMIq0aSgca0uG4uoIm0yadc/WYF/YxQZs9PPFjOT9xadbO9Yh/bs6ZPn4i1Tydrk9PG7pExtLwxuWMFdpuacLC5Ve+uNrsEuSpqiaeLZnHfbhVwIX/0IZaDGXoMAgFPj0ooiiFCq/wQBh2G9fx6ejlXJwa+6zOYFri0BM8EoMlWozhN6dLyqCCPGEj5JqoyfKJ6SdeN0C3oD1fnV9CRSix5gNz/DFnWgGAvRRyoK5bYOsOod4B70Yc9chD8AJmwGSFoA/yuYgaPAZZWum6wHHkuYGdxvOpqh6JklXPuSB1T5YxnY6kwmprobas8usqC9ZINtFjz9nkdzeIi6wkz5Ykwoq6Z1YVCep+DCJWGsk8WbUMdndraC6/KwIx8qeA6Ghwev1Jh2tbIm0j+SiNrrpqTDM+oL0jyiSK1daBI+U6z6+ZpftswyE7phMjuTLg8xjyZ8ZCn7Mli4IXkkCK5OLflXg8o4W4qLs9tlzlI6Y7Mp7kiPHUde/+xhU0XATpkBBVrWgrDAVD6sQ3wM/J/6CV+7y5pT1kCS6nEdG36q49oJv2wTmHitufGUWWohqXUg1bCQTFW8SoI3v7oRyaRtrHFVQH7jhGNsKgR/7bhfpxU9qRTpzUl4kdYmUmhusHq2K6YMwr101AzkLfm3mTKgClVBMa0O8+kG+9SXlApEEy52MtwWgOWPN6cxnDxNmM7Cjm72TdO7pXZxARD1BQ2/Hm/ZVYD/dEnINMv4Utf2wSfhKC9+Sfqf4rR+ZunvFjJYVoUU8n6id+4S6TjzhsDiTkAraE+zClV6tnCETRKLJffLTxvGoGawBmQgRCIW61iPDqd35LVHXacSNgcujuSgZ4QXiK05bP6pnE6kn5t4bzYHXyiTS88CtzQkiM/qabtCegllx+nCOqGR56la+dCOeFc5ZfP5W3NTp+CyTmluq9RWc6eMzoxM61/yKH9+WTta86FiG8XnZFxUFv8/poEzGwcKbdaP1UEYetF//3sh1y6ljTYgiA5n+YkqvNDrUnYCaFUkcnNuPzWpue694JHr/FClynOTrje69s28QTm4U1ebVQktT+nLPlg+O/OyDz3P+0ov5WxSiOC+Ksph9a/8aKOoLZZ/qZ1tA0Wj5RBf9hIN+0bEyde6XzfAumZaZCG+kiAgj5DUDTsvALvoV0tCpq4dpnJx4byoy2zYGNTNbZM4mOXRJ/rBNzUAyl0qM2ot7lSI9OG2+QEuetOnALHtzPht+87Es3ct1i460fvD26piD8uY8dYLYIO5RdGj3XeaOPEyX3mcBnSKeyLXVhJl0vtiVIDJ2wu1sF88q7MyaO/VQ4uEyZw1T8MvjzhxPPzaLEIqOjM59YOnaJ3PfkklkcpoBm6vuaI65G6SzQDExoZWBMSCqQYTAW9qkvQHNtDiJ5YDHWEtayR/N/0Cq9zi14WRWQMi2e7ZajDa6VAypduoxAQiOeMlGDUehazIoiKUx/bODUdmLAEO9FJWE7dsYTzWmIxYT1EpadPNXRvI2ka/cylaRl5rYGnjDIKXRuEtkgqzjd6Xc4rh+W+kNKZl3gvaA63OvPTEtTY/RWluTTSLEDeOHvpKKr/eCUJiYHSICkI4MWz4CnQhJ0PhwfxE8HQqrm2sDnI+dcrQJCBZ8Y/TcrvMPz7ygBwMiJpAIYeBikdds34EHKvrg6hVB80cOnLjHdw0m9+OjvGoA3DPh/lOOBWT7PW8Tg2gxEic7hcVYkUKopA0JZIczgQSb4HWWTvzYgEo4k2/ESUD6bvCCDTXhXC2HKUdezl5YFKGDviQuWYXLtUJZZjYfo3UWGSTKMTCPD2A20BCSSTLBrvc/v6qznewX2zKMHlFwP1LZffDd/VtgPamvTqGcxjJYsZPqlFLN63j+aZQfLpKzfbB/0oHnE0HhFg8Gl8N1063pOqLiAWgqAYkn8GfK3jdMIDlY6K6+D7QUTfWUv/0nNJyKa+jTZExuZSVi2uTon/NvCuxCXjo3PJ5dA/77UeTO4SdtkpBUtYcQIjf6P+0dZDA0QyRs36EgQWRkwmTgT73HEl6YOPFu3Yn1b4Vn7rDo5jSeJHyXN7RjOyz69ZMrP2k2srWyKRcKUdYg4DgrwwQyKWRuIge7kI+vbWCbkyzOjwX1ExsqJkGEqSdkwIIC4nN+H3Pl+CbEsDdrFBt3jOfE/wQWrtmLnoF3PGpaaQBp2rM8Ue7f6KhDDgYqwbQBOOA+qxKakL1lVvkWvBqH9cyvk2CY4UMyKB5YO2QedhXXZmw1YWKTVnsmq9f/xg/ohf4/b7ABp5XdT7ovmRGbGXO717WJ450q/W8SM9EdkLJf4sX1usJaRdtW94HfxdLFdqYZ3kxwpYl8bZ/eJNminPDWAkUR+C+Ufy18UJpx0FAfQJNJPWvgzdhSEPkfZ5c/jYgrNeOPZ91fXurGHHi6hp3m3DWrKxkJob0lnEg/T1I8bAAiFRvadvSaFWWRpQzttJbxsmFDH9i45GX9LRqWAAQQKFPjo6OOJ/uBQEqmdgsE5014dh6JNy5pQsk7Yf+LdJensvRw1tEBmzaGhoADeJVPVfOt9WQXS5t4qtbA4Tk1+32HsCjCEBuDpE7Um3aitjeLgwK76S6OzxkxIVAPV57kSlRxAG+NrhwR15QJ78eGRKrI7AS2MyI9xMc6Wx3H9UAFWn1tjgzKECynSdUEbADuM9k0x0zHvAt4dEAvc9dWVWRw3eIiDao71vvpF2IYfZ0R/3JwgQno72ykyhSJVG9xzl9Ahb3LODKEEvG/rxFRihEtIFADfaQ1NekHiETZgUROQHKMpxtNnB4LbpcLc67hS92Iu8/GiHZGtiM3J7kYvKQgmQcUeR+Gjp5WuJU6uC76jlj+97zMeeReXNfXJBEKLcvFzb6KuGhIzwDArinHFZkbt4MgFuWKCdJ88iBflj0PJY1BzWg1Z0HjlxB4jt7UnhY0WVnmsjQMyr4oYfzZrD4dqiGLTdDWrfk8dphI67l8G0EfdM/9hlGv+PSg69zsIt1KFoqmvvZkIRdVi/+q/A2oss5ZmEz+c0Jfo2hxBDT+9u7R8qZ2Xjj/MNYZgXRihUuW3Y9LS3/SfA4kBHa+1tyha+5CiwfvQJOkV4dF2H3FiVeRwsAOqzks8/kFdPv1cV27f/Ct2AJj1SRgPpeMl/KfXNHfeE+DnFTvbQZws6JaCEeTPxHV7V8jae0Hv80ynIGm32cLB513Gc/+3g0Rk+M4SdvRkxTolsC6pZAM9ZyBMbyDkyssrvwyTcNPdq1oBzfa3vEGks3Zbi9lWc71cDty3N0Ijn31015uOlGgcOwwcDI8G2zYsbPAQKaia4W5l5LEu8I/NB6jLEvBFY9xR0IsclGow+s1QkTnEF0wyPk8dJIEobtkhpdNOr0heTNS0F0iI/rcGl9QTtVJGRxA01mAY1POvhRsWW0oCJkpxJ6ljlGs2LI2Sl+wHlWerEjMgs/cSWgqlsfHBz8azo7Z7T3MRJ+FotJnxYYFr1ENLu+Op1dYdcgHQVcySlhP6n1s2nw9wL4l/GBLGV3gTn0DH6bPOsPnEaLqzWwIGuUEhu5eshKhwQrYIpiMcdc91a2KMnLxwpv3r/mOsWY7iG3Kz7bYtV4Dyc0A+9/kLhU2KHbh+rHIZB71Nb1ldXpL0ziO2sJidfvzg8eJCy63bQ+Qt1WD7K+XsfO20pt/78EyQLyfGImLa64gpgP/73vCmbal10DHyB++M+rj+/9E5gl9kXwz5o5RSRJJeQvc0ST6D6IkkDBhaOx9zteLXmjOB8r2uMwPLdd+iMPHS+/GekcbpezOi+aj4kbqpMdT9H/XKoPz6ZBdNCsuVdHGqQtbQADuryU2kdCYmBswNAZVOdLmVHbcQDR5m43lK/i9t13u0ahqf7CL7PhMLiatcTNW4GIudmMuWbIV+sTADOGITK6sHNLMnFl6GV2HReE1iYox3VwFD8jPiguHQJegk0EKj1jjDxE+Yh53K8DD8/RCeo6noUL0YK7eJzasClcj5Yd1rSJDxmbCi4ubzHionaBVdFWS+iAcZMbktEFKpswR7x6tdJDE7rFdJw96z7Hc/Vi3JTostEWU0/z2wyKjCQisJTauYAVinbZ4tZg4n8AvJYii9b0yI9dBQOXz+IAkwFZti11LA6zZ/eRyNOlomwbhjsZikfX9y2shMzba4f+g3k8BU78RX6Opj3LD1kewbXCbtYPCHK0WBx/vUrWgmWOqUy2TN+l9yUTWDq02Cd1VAiAEcuFkQly4/SKAIgyYdNwXgTl6bnrxH6D+7Mb0hSO/EfgGdPjzLnf4W4ZmkGDn/scGGSjZ6hyumflO1DD4gaL6aS5dX6lt5g4ckBCVmoCfTEJ1irpi8WlzcYjf0lsfAtPUyOzM9AbUqms9rri9WmRsPU+BUbSiRU8W2yCxBqDV1iWsrydfUPt8+K8DnMMeaPfvKTuK80+yKjNPMeLIi7LJYa3+YvuhZycRmVjKeDyBFEn+mnVG0/hzhIOdTt+lNp+f1RtMzQ2d0GmYseUxLUhJdnHlWDI60oHA9QC42IXp1tivT/c1PO39zoU0sxwTUvn0PB2JmnKWuZkuuIP6/T7Jku7pya5tEA7yRHcHG6CeVx71D27EP5R2LZhJzjfPRBf0T6EDKr5BCj62YbzRCuXsYc2E2t3/278tCp2AqotznmO5lgo2XsfQu3VSa63Nyw7pB9dO1l8llMwiMHh6DOiWf8BhfMlYQXNcGov7udm/a/T2KrcNpluD3v4quc/sNKtQnHdivQrUhEUlrSQ/m357MjwwBXZhI1P2fo76fstcom7F4zO9JHgUytDSchVi+MyY2enrG9PcQvqqQyo41SMhMcadJMuwzwy6Nkel2Yfpc0Pu3u6T4KUBtgYV9/bpslqQpZgs5sfgdavmWyD3kY8Z2roI0aKz5/g3MUYsWlzpiqY7n5oOuf4DJh5cLUxxaOuQ5elCxoC8cDthhW38KjbktWwI9DMmbklh2PBSJ+1W1zHVU4bIr+1SB6BHdgloZ/UJ5/7rqx9urUqOvcCarec+M2azHES5IUKZBF+BkHeVEVQFY8ik+Nadp7jnt81yiPKT0mE2a8ieEaVcYv67JrH9YNsF4wA3Kxo1AaA3zJWfdCjC4VljyPq5CTHZfUbIs9XDMnoS8yApVzCxkncu0EE3Zzido6E9ekL3JpJpByLdyP0NnQfjLnoZTLBL8E8Mkx0duYvoBRy40kzu2c/YkQzsuPWH93krGYcuDWy4ew54/c2C7+oaQN9+ZuKROrX4a0JMtDxNjWgBTRGYHTdSyLnK+ZWCGJWZQRFYIqaQMNQpGYU57d/Jv43WLt1kkmotrz+1K+l9fB+eJWsgafbLoXvQ1bP2ClO/1xk4HU2LMQV2j1KjcMGbh6Rk5cL6rxQU+uA8UDUhb/BTTWbLqXj0wgf4/TYP0PHoaCcCHN6wnYBOsnBN0putR2MGHtmYVuSEIS1Blsyz81p+hc0taKRqNC4LryYv2lyZbh+celMJmolOdLl0WgNb4GgsGbagVB0EDbv59IFygCYdX3+NJcgr3V6++uS6CT5wWnISmiCYpd4UfxaDSok7uP6rJVCG5rZmbw8XAOdewoRid6KW+yhbMKallRdpK9aOnMIY0BxfKTrh8O+RdDnSg39J3LxGd6ruIQ5I4Qy49ebOIkMQbanYyY9yqlXUvo10P7K9l3Ll1n0LvgrRWteMnzywZ+/K41C2eBSydngdIdqzIMKASmH+PZraQ8CzJfW0BRFlBJ2MyJHOrkSIvlL5b5Yytj7xFCY2L+hR8Xz6Ley7bSRQvFL4MKsoBDx+75pHvHwnIyf4D6/A2bUCVESnFYLdr5osxgHZUuIuymHMZ9F+NruSLjhDatsN8i83cR+bupu05cxwx1Uu8Cc+qNtrBGZgiVzTa9Xnl7QiZBouGxjrPKRlDw6/rpzMBBpIKFrepZuYzw/wDmbk4SAMgBkmt14pOz8kflqyIJ2M9JuC5rzrhGp0CRV+yEkdIH/K3WLrh/r/S+iy4y1onnXLO5BIwXyl28gQvT62J/nFKJEzi9WhIDLRlheKOE3EiFAC0UW79RfLOHunErq3o299ZZGBnsbRIePqPo7/+NSNpErceSZ3Aqm8FBchhdvTq5IGODvr4XCXgU+nXyX7M9BG82UEd1pf+F4frg+b2FYmqAh9jlQDAbKN4pKpsETNmvPhnzup5yONYokBYL7CV0C0/JWPZDv783j8my8kbgTJpeaRvA4xXLbg/4p2jtyF2KA5WnvHjgzwmpFXFOGTkZN/+smrP+Y5D2+m6onNs7hes6M9HmJq1wWRZHeFu8jAXnxFnbq6wcDXWanoYtXKz1kIBf7Cn8UpRhJi1ik26YgIHVK8isc8vIK5QbUryuncmFo2KVB48sTNx++8eYX3kgkb/7VWN5gZcPtQemTEBrcunXR9W811CYe7Y+EcCUS6OOcPR6eOyYZ2cm9ATx6gbpM3z2Z+MOd8Xnr6gsosC8dBEm7IpcN5Yas7rgp2bADm7yDWeIivOQszGCri9BsMCiZ0MfjKHN39xeST7u2/3KlqrDswx8WAVtjK32J/jYMQWif5vtbvKG1ogV36ie0suzOd7dvvouaOr3kd1D8D2IGrHlX0XiMpoEq/OgGcZdEkPxn+XT3cjVZQtZ85bh+PUSq/MAg0M9D2AU3qjm/zfWxwgnQSTCqiAeP/kPnu+00Tl1LPp064ZWoNukWKZe08aBoBXLL9rJqLXMBIs+ibJyyJ6k9zM+LDgFxOmO8vNpNGYKg73UHpLLZPUApZknoPnKcmqfUpMqZk9wu/KXwvfjuKSdbNsbOt9vvCi3nV0lAALf3uGajOfVuDb/Swb1PwaPgfKDG+z+bP++WJGL1BewoGE5tGVBw8GyGjc1/I58MoUmXJ8YrGBVjEldjW+d2yoVeS2zqux/bbhEAGEHwvjivb45vku65/podKGYijo189nshCPFEBwoFVJX0qtX9G31OZln4uD3goa0/OQBUjD+7NRxVaNdw90+3cE7KoxPi7eFNDTwMCegBTSlnv86zTskvaynyDkdLElpmjLyOXbhjn5w5lOKWNGitZUq248OYh/2hRfqrbHmX5Lb6XB5iY7xwzLGbqIN9V+OShy9JhHMq5tvMlpVotJvgrb+fheja/TUwFIB8MN2lxFBNFDLmPxN1GjiWljENvjnubd3kMwYDYjF8ErebYi2DukwG/LQHhi5KuzpOwuRcK6gzu0BYctx3/TCFuJUEik55m9mjwPFj4BFC0N1Xw+QhoTaBcpts1Wh7L+fWRsaurc5GZJKbokv20NvNKgV941rvTUwGSwGBB8I9TMj2fiQ3X4Y0UgA5l0TgJUn21Y3sG793uVfb+bFhIeBGB6BIKRENwnxCCFpZhnTNFMWOEcvTSFkUx+secsE010oSlB+6nmJd5/7XLfHoBtxcBTv2k5LdMlef5onM4b7poJbrc9PANbNSRbQWbhcIP+6F06TCrN9nbOvX9O9Y/RvDchg2mdvD/MZNSz5cMvVav7geJK3VYWKLCgbBbAfb3speBqowF56vKbNI/SGvz+GcX2tLy6UGRuDx+lPS7jWAp+SJrqsD6ThL5w/MhxGph33x6fHJNdk7suMOm2/noRxvOkDZaxSWBKWYVBU+4LB6Ci9FEvpcncWn6f91aB//Ab5U4mi9mmZtyoYNeOZDqj3+kIPmtcVhHBURVDTY05hWH0lUYyGAEht54yyvecqrzXcz0q7hoDJlq3zF7cZ77wL4HofwMsJKuAizhPwrUGxsGKigeyMWRDOEtEQBZNAyDUQ8tzCBpnVeTh3MTCpXCMFE6PYuRnrJhhTRmDk7gsqNZcacrC+8j4Z4ZdBFRlf9dIJalkpII6C3HjZ8pEqjcJlUb+HImxJ5Ia3gbRnn607xewkuysaO8LdvFvOSuLCOam3nQSeBeleWEuFIyw6BpoGm5YYpDL38lUujLe6rk1e5ki0noolJlRIK8KmTFkCMVr6uY2662+GJuDSczRkp9MDSt4aC0oqGbak89Oe3wbDPeAspriUtElxl+aDk4Yq7k2OrBebUstri6lNMOVbRw+fEbmqbePhP0mdd8J1Qzt9OArsvpA0kcNWx2iDDJc+h1oEUPssFPs+MbZAf+u/D3DwLR/2/yRp4Cx/hhhCFeVMPO9zy2r+K6mlOoGgElu3c1fc+6pfpTy5rZaBdo5J9n4W6/ZTnbbfSdS3KkAMLpU7LEJ2UQPK9+cwifRlFK+6iUtwyZ99mv9ubz/0W6mYLYe0kozqaWn7uXI96vTfoyWT68BN6Ze/RdcbUsq1g0vCg7kizIpGf7A+Pg4cBZ5et2r7ofmvTKenWkfF7d6HH+l53UulBpCyyPD+eakgVF8k+un7alXh7saGt5DBLiS3Vn17v7x0gsJl"
r = agent.post(
"http://epbcnotices.environment.gov.au/_services/entity-grid-data.json/#{id}",
{
"base64SecureConfiguration": base64_secure_configuration,
"sortExpression":"new_dateofnotice DESC",
"search":nil,
"page":page,
"pageSize":30,
"filter":nil,
# Only get last 60 days
"metaFilter":"0=&1=&2=&3=2"
}.to_json,
"Content-Type" => "application/json; charset=utf-8"
)
data = JSON.parse(r.body)
data["Records"].each do |record|
r = {}
record["Attributes"].each do |a|
key = case a["Name"]
when "new_epbcnumber"
"reference_number"
when "new_titleforweb"
"title"
when "new_webdisplayname"
"notification_type"
when "new_dateofnotice"
"date"
when "new_casedecisionid"
"case_decision_id"
when "a_2db5612c0696e41180d9000d3ad00a6f.new_sectionoftheact"
"section_of_act"
else
raise "Unexpected name #{a['Name']}"
end
r[key] = a["DisplayValue"]
end
# Now get the attachments
result = agent.post(
"http://epbcnotices.environment.gov.au/_services/entity-notes/#{id}",
{
"regarding":{
"Id": r["case_decision_id"],
"LogicalName":"new_casedecision",
},
"page":1,
"pageSize":20
}.to_json,
"Content-Type" => "application/json; charset=utf-8"
)
JSON.parse(result.body)["Records"].each_with_index do |record, i|
r["attachment#{i+1}_name"] = record["AttachmentFileName"]
r["attachment#{i+1}_url"] = (result.uri + record["AttachmentUrl"]).to_s
end
ScraperWiki.save_sqlite(%w[reference_number], r)
end
# Return total number of pages
data["PageCount"]
end
agent = Mechanize.new
puts "Getting data for the last 60 days"
page = 1
puts "Page #{page}..."
page_count = scrape_page(agent, page)
page += 1
while page <= page_count
puts "Page #{page}..."
page_count = scrape_page(agent, page)
page += 1
end