-
Notifications
You must be signed in to change notification settings - Fork 0
/
Amazon Scraping_AmandaLuz.py
465 lines (465 loc) · 18.2 KB
/
Amazon Scraping_AmandaLuz.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"id": "a02a5d4e",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"ESCOPO DO PROJETO\n",
"#Abrir navegador em background\n",
"#Acessar a Amazon, fazer a busca por Iphone\n",
"#Obtém a URL do resultado\n",
"https://www.amazon.com.br/s?k=Iphone&__mk_pt_BR=%C3%85M%C3%85%C5%BD%C3%95%C3%91&ref=nb_sb_noss\n",
"#Capturar o conteúdo da página\n",
"#Segmenta o conteudo obtido\n",
"#Montagem da tabela com os produtos da primeira página\n",
"#Obtém diretório para salvar o arquivo xlsx\n",
"#Gera a planilha a partir dos dados obtidos usando um dataframe e depois convertendo em excel\n",
"Planilha salva com sucesso no diretório - C:\\Users\\amand\\DadosAmazon.xlsx\n",
"#Fechando navegador\n",
"#Abrindo a planilha e lançando para a tela\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>NOME</th>\n",
" <th>PRECO</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Iphone 11 Apple Preto, 128gb Desbloqueado - Mh...</td>\n",
" <td>R$ 5.069,00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Apple iPhone 8 Plus, 64GB, Gold</td>\n",
" <td>R$ 3.898,00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Iphone 11 Apple Preto, 64gb Desbloqueado - Mhd...</td>\n",
" <td>R$ 4.519,00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Novo Apple iPhone 12 Pro Max (128 GB, Dourado)</td>\n",
" <td>R$ 8.639,90</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Iphone 11 Apple Branco, 64gb Desbloqueado - Mh...</td>\n",
" <td>R$ 4.489,00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>iPhone 11 Branco, com Tela de 6,1, 4G, 128 GB ...</td>\n",
" <td>R$ 5.065,00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>Novo Apple iPhone 12 Pro Max (128 GB, Azul Pac...</td>\n",
" <td>R$ 8.472,75</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>Iphone 12 Pro Apple Azul-pacífico, 128gb Desbl...</td>\n",
" <td>R$ 8.169,90</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>Iphone Xr Apple Branco, 128gb Desbloqueado - M...</td>\n",
" <td>R$ 4.199,00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>Celular Apple iPhone 11 64gb / Tela 6.1'' / 12...</td>\n",
" <td>R$ 4.499,90</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>Novo Apple iPhone 12 (128 GB, Branco)</td>\n",
" <td>R$ 6.695,00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>Apple Iphone 12 (128GB Preto) - Desbloqueado A...</td>\n",
" <td>R$ 6.583,71</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>Novo Apple iPhone 12 Pro (128 GB, Grafite)</td>\n",
" <td>R$ 8.218,48</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>Carregador Original iPhone 12 Turbo Usb-c 20w ...</td>\n",
" <td>R$ 145,00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>Xiaomi Redmi Note 9 128GB 4GB RAM - Versión Gl...</td>\n",
" <td>R$ 1.295,97</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>Carregador Universal Ultra Rápido Duo, 1 X USB...</td>\n",
" <td>R$ 94,63</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>Novo Apple iPhone 12 (128 GB, Azul)</td>\n",
" <td>Indisponível</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>Novo Apple iPhone 12 Pro Max (128 GB, Grafite)</td>\n",
" <td>R$ 8.199,90</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>Redmi Note 10 128GB 4GB RAM - Onyx Gray</td>\n",
" <td>R$ 1.659,00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>Capa Capinha Protetora Para Iphone 12 e 12 Pro...</td>\n",
" <td>R$ 54,99</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>Carregador de Iphone Original 20w Usb-c SE XR ...</td>\n",
" <td>R$ 68,00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>21</th>\n",
" <td>Fone de Ouvido Apple EarPods Lightning</td>\n",
" <td>R$ 172,85</td>\n",
" </tr>\n",
" <tr>\n",
" <th>22</th>\n",
" <td>iPhone 8 Plus Apple 64GB Cinza Espacial Tela R...</td>\n",
" <td>Indisponível</td>\n",
" </tr>\n",
" <tr>\n",
" <th>23</th>\n",
" <td>iPhone SE Branco, com Tela de 4,7, 4G, 64 GB e...</td>\n",
" <td>R$ 2.999,90</td>\n",
" </tr>\n",
" <tr>\n",
" <th>24</th>\n",
" <td>Novo Apple iPhone 12 Pro Max (256 GB, Azul Pac...</td>\n",
" <td>R$ 9.859,00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25</th>\n",
" <td>Pelicula Vidro Temperado Hprime Apple iPhone 1...</td>\n",
" <td>R$ 28,90</td>\n",
" </tr>\n",
" <tr>\n",
" <th>26</th>\n",
" <td>Iphone Xr Apple Preto, 64gb Desbloqueado - Mry...</td>\n",
" <td>R$ 3.899,00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>27</th>\n",
" <td>Pelicula NanoShield para Apple iPhone 7/8, HPr...</td>\n",
" <td>R$ 23,99</td>\n",
" </tr>\n",
" <tr>\n",
" <th>28</th>\n",
" <td>Novo Apple iPhone 12 Pro Max (256 GB, Grafite)</td>\n",
" <td>R$ 9.299,00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>29</th>\n",
" <td>Iphone Xr Apple Preto, 64gb Desbloqueado - Mh6...</td>\n",
" <td>R$ 3.744,98</td>\n",
" </tr>\n",
" <tr>\n",
" <th>30</th>\n",
" <td>Smartphone, Apple, iPhone 7 MN952BR/A, 128 GB,...</td>\n",
" <td>R$ 2.349,99</td>\n",
" </tr>\n",
" <tr>\n",
" <th>31</th>\n",
" <td>Novo Apple iPhone 12 Pro Max (256 GB, Dourado)</td>\n",
" <td>R$ 9.134,90</td>\n",
" </tr>\n",
" <tr>\n",
" <th>32</th>\n",
" <td>IPHONE 8 64 GB VITRINE. 100% SEM DETALHES.CAIX...</td>\n",
" <td>Indisponível</td>\n",
" </tr>\n",
" <tr>\n",
" <th>33</th>\n",
" <td>Iphone Xr Apple (product) Vermelho, 64gb Desbl...</td>\n",
" <td>R$ 4.305,53</td>\n",
" </tr>\n",
" <tr>\n",
" <th>34</th>\n",
" <td>Kit Capa Anti Impacto e Película de Vidro iPho...</td>\n",
" <td>R$ 39,90</td>\n",
" </tr>\n",
" <tr>\n",
" <th>35</th>\n",
" <td>CAPA CASE CAPINHA SILICONE AVELUDADO IPHONE 11...</td>\n",
" <td>R$ 42,89</td>\n",
" </tr>\n",
" <tr>\n",
" <th>36</th>\n",
" <td>Carregador turbo 20W USB-C mais cabo USB-C DE ...</td>\n",
" <td>R$ 65,00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>37</th>\n",
" <td>iPhone 8 256GB Apple Tela 4.7 iOS 11 Câmera 12...</td>\n",
" <td>Indisponível</td>\n",
" </tr>\n",
" <tr>\n",
" <th>38</th>\n",
" <td>Spigen Capa Liquid Air Armor Projectada para A...</td>\n",
" <td>R$ 189,90</td>\n",
" </tr>\n",
" <tr>\n",
" <th>39</th>\n",
" <td>Iphone 11 Apple Vermelho, 64gb Desbloqueado - ...</td>\n",
" <td>Indisponível</td>\n",
" </tr>\n",
" <tr>\n",
" <th>40</th>\n",
" <td>Pelicula de Vidro Temperado 9h para Apple iPho...</td>\n",
" <td>R$ 23,99</td>\n",
" </tr>\n",
" <tr>\n",
" <th>41</th>\n",
" <td>Película De Privacidade Vidro Temperado Para A...</td>\n",
" <td>R$ 49,99</td>\n",
" </tr>\n",
" <tr>\n",
" <th>42</th>\n",
" <td>Iphone Xr Apple Preto, 128gb Desbloqueado - Mr...</td>\n",
" <td>Indisponível</td>\n",
" </tr>\n",
" <tr>\n",
" <th>43</th>\n",
" <td>iPhone XS 64GB Dourado (Dourado)</td>\n",
" <td>Indisponível</td>\n",
" </tr>\n",
" <tr>\n",
" <th>44</th>\n",
" <td>Kit 2x Películas Vidro Transparente Câmera Len...</td>\n",
" <td>R$ 29,99</td>\n",
" </tr>\n",
" <tr>\n",
" <th>45</th>\n",
" <td>iPhone SE 64GB Black Novo Desbloqueado Tela 4,...</td>\n",
" <td>R$ 2.949,00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>46</th>\n",
" <td>SGVAHY Bling Diamond Case para iPhone 7 Plus/8...</td>\n",
" <td>Indisponível</td>\n",
" </tr>\n",
" <tr>\n",
" <th>47</th>\n",
" <td>iPhone XR 128GB Apple iOS 12 4G + Wi-Fi Câmera...</td>\n",
" <td>Indisponível</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" NOME PRECO\n",
"0 Iphone 11 Apple Preto, 128gb Desbloqueado - Mh... R$ 5.069,00\n",
"1 Apple iPhone 8 Plus, 64GB, Gold R$ 3.898,00\n",
"2 Iphone 11 Apple Preto, 64gb Desbloqueado - Mhd... R$ 4.519,00\n",
"3 Novo Apple iPhone 12 Pro Max (128 GB, Dourado) R$ 8.639,90\n",
"4 Iphone 11 Apple Branco, 64gb Desbloqueado - Mh... R$ 4.489,00\n",
"5 iPhone 11 Branco, com Tela de 6,1, 4G, 128 GB ... R$ 5.065,00\n",
"6 Novo Apple iPhone 12 Pro Max (128 GB, Azul Pac... R$ 8.472,75\n",
"7 Iphone 12 Pro Apple Azul-pacífico, 128gb Desbl... R$ 8.169,90\n",
"8 Iphone Xr Apple Branco, 128gb Desbloqueado - M... R$ 4.199,00\n",
"9 Celular Apple iPhone 11 64gb / Tela 6.1'' / 12... R$ 4.499,90\n",
"10 Novo Apple iPhone 12 (128 GB, Branco) R$ 6.695,00\n",
"11 Apple Iphone 12 (128GB Preto) - Desbloqueado A... R$ 6.583,71\n",
"12 Novo Apple iPhone 12 Pro (128 GB, Grafite) R$ 8.218,48\n",
"13 Carregador Original iPhone 12 Turbo Usb-c 20w ... R$ 145,00\n",
"14 Xiaomi Redmi Note 9 128GB 4GB RAM - Versión Gl... R$ 1.295,97\n",
"15 Carregador Universal Ultra Rápido Duo, 1 X USB... R$ 94,63\n",
"16 Novo Apple iPhone 12 (128 GB, Azul) Indisponível\n",
"17 Novo Apple iPhone 12 Pro Max (128 GB, Grafite) R$ 8.199,90\n",
"18 Redmi Note 10 128GB 4GB RAM - Onyx Gray R$ 1.659,00\n",
"19 Capa Capinha Protetora Para Iphone 12 e 12 Pro... R$ 54,99\n",
"20 Carregador de Iphone Original 20w Usb-c SE XR ... R$ 68,00\n",
"21 Fone de Ouvido Apple EarPods Lightning R$ 172,85\n",
"22 iPhone 8 Plus Apple 64GB Cinza Espacial Tela R... Indisponível\n",
"23 iPhone SE Branco, com Tela de 4,7, 4G, 64 GB e... R$ 2.999,90\n",
"24 Novo Apple iPhone 12 Pro Max (256 GB, Azul Pac... R$ 9.859,00\n",
"25 Pelicula Vidro Temperado Hprime Apple iPhone 1... R$ 28,90\n",
"26 Iphone Xr Apple Preto, 64gb Desbloqueado - Mry... R$ 3.899,00\n",
"27 Pelicula NanoShield para Apple iPhone 7/8, HPr... R$ 23,99\n",
"28 Novo Apple iPhone 12 Pro Max (256 GB, Grafite) R$ 9.299,00\n",
"29 Iphone Xr Apple Preto, 64gb Desbloqueado - Mh6... R$ 3.744,98\n",
"30 Smartphone, Apple, iPhone 7 MN952BR/A, 128 GB,... R$ 2.349,99\n",
"31 Novo Apple iPhone 12 Pro Max (256 GB, Dourado) R$ 9.134,90\n",
"32 IPHONE 8 64 GB VITRINE. 100% SEM DETALHES.CAIX... Indisponível\n",
"33 Iphone Xr Apple (product) Vermelho, 64gb Desbl... R$ 4.305,53\n",
"34 Kit Capa Anti Impacto e Película de Vidro iPho... R$ 39,90\n",
"35 CAPA CASE CAPINHA SILICONE AVELUDADO IPHONE 11... R$ 42,89\n",
"36 Carregador turbo 20W USB-C mais cabo USB-C DE ... R$ 65,00\n",
"37 iPhone 8 256GB Apple Tela 4.7 iOS 11 Câmera 12... Indisponível\n",
"38 Spigen Capa Liquid Air Armor Projectada para A... R$ 189,90\n",
"39 Iphone 11 Apple Vermelho, 64gb Desbloqueado - ... Indisponível\n",
"40 Pelicula de Vidro Temperado 9h para Apple iPho... R$ 23,99\n",
"41 Película De Privacidade Vidro Temperado Para A... R$ 49,99\n",
"42 Iphone Xr Apple Preto, 128gb Desbloqueado - Mr... Indisponível\n",
"43 iPhone XS 64GB Dourado (Dourado) Indisponível\n",
"44 Kit 2x Películas Vidro Transparente Câmera Len... R$ 29,99\n",
"45 iPhone SE 64GB Black Novo Desbloqueado Tela 4,... R$ 2.949,00\n",
"46 SGVAHY Bling Diamond Case para iPhone 7 Plus/8... Indisponível\n",
"47 iPhone XR 128GB Apple iOS 12 4G + Wi-Fi Câmera... Indisponível"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import pandas as pd\n",
"import time\n",
"import openpyxl\n",
"import requests\n",
"import lxml\n",
"from os import path\n",
"from bs4 import BeautifulSoup\n",
"from selenium import webdriver #caso utilize o Firefox baixar o geckodriver\n",
"from selenium.webdriver.chrome.options import Options\n",
"from selenium.webdriver.common.keys import Keys\n",
"\n",
"while True:\n",
" print('ESCOPO DO PROJETO')\n",
" print('#Abrir navegador em background')\n",
"\n",
" options = webdriver.ChromeOptions()\n",
" options.add_argument(\"--headless\")\n",
" navegador = webdriver.Chrome(options=options)\n",
" #navegador = webdriver.Chrome() CASO QUEIRA VER O NAVEGADOR ABERTO, TROQUE A OPÇÃO ANTERIOR POR ESTA.\n",
" #navegador = webdriver.Firefox() \n",
"\n",
" print('#Acessar a Amazon, fazer a busca por Iphone')\n",
" navegador.get('https://www.amazon.com.br/')\n",
" navegador.find_element_by_xpath('//*[@id=\"twotabsearchtextbox\"]').send_keys('Iphone' + Keys.RETURN)\n",
" time.sleep(1) #pausas para garantir o carregamento da página\n",
"\n",
" print('#Obtém a URL do resultado')\n",
" linkBusca = navegador.current_url\n",
" print(linkBusca)\n",
"\n",
" print('#Capturar o conteúdo da página')\n",
" res = requests.get(linkBusca)\n",
" res.encoding = 'utf-8'\n",
" time.sleep(2)\n",
"\n",
" soup = BeautifulSoup(res.text, 'html.parser')\n",
" print('#Segmenta o conteudo obtido')\n",
" #soup = BeautifulSoup(res.text, 'html.parser')\n",
" time.sleep(2)\n",
"\n",
" print('#Montagem da tabela com os produtos da primeira página')\n",
" planilha = []\n",
" produto = soup.find_all(class_ = 'a-section a-spacing-medium')\n",
" \n",
" if not produto:\n",
" print('FALHA NO SITE PESQUISADO! Tentando novamente!')\n",
" navegador.quit()\n",
" continue\n",
" else:\n",
" break\n",
"\n",
"for a in produto:\n",
" info = a.find(class_ = 'a-section a-spacing-none')\n",
" title = info.h2.text\n",
" preco = info.find(class_ = 'a-offscreen')\n",
" if preco:\n",
" planilha.append({'NOME':title,'PRECO':preco.text})\n",
" else:\n",
" planilha.append({'NOME':title, 'PRECO':'Indisponível'})\n",
"#\n",
"time.sleep(1)\n",
"print('#Obtém diretório para salvar o arquivo xlsx')\n",
"pastaUsuario = path.join(path.expanduser('~'))\n",
"\n",
"print('#Gera a planilha a partir dos dados obtidos usando um dataframe e depois convertendo em excel')\n",
"df=pd.DataFrame(data=planilha)\n",
"df\n",
"df.to_excel(pastaUsuario+'\\DadosAmazon.xlsx', index=False)\n",
"print('Planilha salva com sucesso no diretório - ' + pastaUsuario + '\\DadosAmazon.xlsx')\n",
"\n",
"\n",
"print('#Fechando navegador')\n",
"navegador.quit()\n",
"\n",
"print('#Abrindo a planilha e lançando para a tela')\n",
"Dados = pd.read_excel(pastaUsuario+'\\DadosAmazon.xlsx')\n",
"display(Dados)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8bf55c5c",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.8"
}
},
"nbformat": 4,
"nbformat_minor": 5
}