[search] fix it again

BrightDV · Feb 21, 2024 · 293db6e · 293db6e
1 parent b81973c
commit 293db6e
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 27 deletions.
diff --git a/README.md b/README.md
@@ -59,7 +59,7 @@ If you want, you can know where he is born and other personal info (not very use
 | :---------------: |:---------------:| :---------------:|
 | Home News | Formula 1 API | https://api.formula1.com |
 | Home Videos | Formula 1 API | https://api.formula1.com |
-| Articles search | SearXNG | [9 instances](lib/api/searx.dart#L26) |
+| Articles search | SearXNG | [14 instances](lib/api/searx.dart#L26) |
 | Standings (Q, S and R) | Ergast API | https://ergast.com/mrd |
 | Standings (FP, Q, S and R) | Formula 1 Archives | https://formula1.com |
 | Schedule | Ergast API | https://ergast.com/mrd |

diff --git a/lib/api/searx.dart b/lib/api/searx.dart
@@ -25,9 +25,6 @@ import 'package:http/http.dart' as http;
 
 class SearXSearch {
  final List<String> instances = [
- 'https://search.sapti.me',
- 'https://searx.fmac.xyz',
- 'https://priv.au',
  'https://searx.be',
  'https://searx.tiekoetter.com',
  'https://searx.work',
@@ -36,17 +33,21 @@ class SearXSearch {
  'https://search.demoniak.ch',
  'https://northboot.xyz',
  'https://search.in.projectsegfau.lt',
+ 'https://searxng.site/',
+ 'https://search.broker/',
+ 'https://search.inetol.net/',
+ 'https://opnxng.com/',
  ];
 
  final List<String> instancesWithJsonFormat = [
  'https://search.neet.works',
- 'https://searx.prvcy.eu',
  'https://etsi.me',
  'https://search.projectsegfau.lt',
  ];
 
  Future<List> searchArticles(String query) async {
- List results = await searchArticlesWithoutScraping(query);
+ List results = [];
+ await searchArticlesWithoutScraping(query);
  if (results.isEmpty) {
  results = await searchArticlesWithScraping(query);
  }
@@ -62,28 +63,36 @@ class SearXSearch {
  '$instance/search?q="https://formula1.com/en/latest/article" $query&language=all&format=json',
  ),
  headers: {
- 'user-agent':
- 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36',
+ 'User-Agent':
+ 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0',
+ 'Accept':
+ 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
  },
  );
- if (response.body != 'Too Many Requests' &&
- response.body != 'Rate limit exceeded' &&
+ if (response.body.trim() != 'Too Many Requests' &&
+ response.body.trim() != 'Rate limit exceeded' &&
  response.statusCode == 200) {
  break;
  }
  }
- Map responseAsJson = jsonDecode(utf8.decode(response.bodyBytes));
- for (var result in responseAsJson['results']) {
- results.add(
- {
- 'url': result['url'],
- 'title': result['title'],
- 'content': result['content'],
- },
- );
- }
 
- return results;
+ if (response.body.trim() == 'Too Many Requests' ||
+ response.body.trim() != 'Rate limit exceeded') {
+ return [];
+ } else {
+ Map responseAsJson = jsonDecode(utf8.decode(response.bodyBytes));
+ for (var result in responseAsJson['results']) {
+ results.add(
+ {
+ 'url': result['url'],
+ 'title': result['title'],
+ 'content': result['content'],
+ },
+ );
+ }
+
+ return results;
+ }
  }
 
  Future<List> searchArticlesWithScraping(String query) async {
@@ -92,17 +101,19 @@ class SearXSearch {
  instances.shuffle();
  for (String instance in instances) {
  url = Uri.parse(
- '$instance/search?q="https://formula1.com/en/latest/article" $query&language=all',
+ '$instance/search?q="https://formula1.com/en/latest/article" $query&language=all&theme=simple',
  );
  response = await http.get(
  url,
  headers: {
- 'user-agent':
- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
+ 'User-Agent':
+ 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0',
+ 'Accept':
+ 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
  },
  );
- if (response.body != 'Too Many Requests' &&
- response.body != 'Rate limit exceeded' &&
+ if (response.body.trim() != 'Too Many Requests' &&
+ response.body.trim() != 'Rate limit exceeded' &&
  response.statusCode == 200) {
  break;
  }
@@ -111,7 +122,7 @@ class SearXSearch {
  utf8.decode(response.bodyBytes),
  );
  List<Map> results = [];
- List<dom.Element> tempResults = document.getElementsByTagName('article');
+ List<dom.Element> tempResults = document.getElementsByClassName('result');
  for (var element in tempResults) {
  if (element.firstChild!.attributes['href']!
  .contains('formula1.com/en/latest/article')) {