Issue #13 - Ignoring content for non-HTML files.

classes/robot/crawler.php

@@ -737,17 +737,18 @@ public function scrape($url) {
             curl_close($s);
             return $result;
         }
+        // See http://stackoverflow.com/questions/9351694/setting-php-default-encoding-to-utf-8 for more.
+        unset($charset);
+        $contenttype = curl_getinfo($s, CURLINFO_CONTENT_TYPE);
+        $ishtml = (strpos($contenttype, 'text/html') === 0); // Related to Issue #13.
+
         $headersize = curl_getinfo($s, CURLINFO_HEADER_SIZE);
         $headers = substr($raw, 0, $headersize);
         $header = strtok($headers, "\n");
         $result->httpmsg          = explode(" ", $header, 3)[2];
-        $result->contents         = substr($raw, $headersize);
+        $result->contents         = $ishtml ? substr($raw, $headersize) : '';
         $data = $result->contents;
 
-        // See http://stackoverflow.com/questions/9351694/setting-php-default-encoding-to-utf-8 for more.
-        unset($charset);
-        $contenttype = curl_getinfo($s, CURLINFO_CONTENT_TYPE);
-
         /* 1: HTTP Content-Type: header */
         preg_match( '@([\w/+]+)(;\s*charset=(\S+))?@i', $contenttype, $matches );
         if ( isset( $matches[3] ) ) {

version.php

@@ -26,8 +26,8 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-$plugin->version   = 2016081601;        // The current plugin version (Date: YYYYMMDDXX)
-$plugin->release   = 2016081601;        // The current plugin version (Date: YYYYMMDDXX)
+$plugin->version   = 2016121300;        // The current plugin version (Date: YYYYMMDDXX)
+$plugin->release   = 2016121300;        // The current plugin version (Date: YYYYMMDDXX)
 $plugin->requires  = 2013110500;        // Requires this Moodle version.
 $plugin->component = 'tool_crawler'; // To check on upgrade, that module sits in correct place.
 $plugin->maturity  = MATURITY_STABLE;