Merge branch 'v3-develop' of https://github.com/tonyofrancis/Fetch in…

…to v3-develop

CHANGELOG

@@ -1,3 +1,14 @@
+Version 3.0.8/  Android x version 3.1.2
+1. Fixed group error reporting bugs with FetchGroup.getProgress() method.
+2. Fixed network on main thread error.
+3. Improved network connection checks.
+4. Improvements and fixes to the StorageResolver when working with the Storage Access Framework.
+5. Files are now pre allocated on local storage on request enqueue. This prevents waste of data. See FetchConfiguration method `fun preAllocateFileOnCreation(preAllocateFile: Boolean)` to enable or disable this feature. On by default.
+6. Added new field segment to Downloader.Request. See java docs for details.
+7. Added new Fetch method `fun getContentLengthForRequests(requests:List<Request>, fromServer: Boolean, func: Func<List<Pair<Request,Long>>>, func2: Func<List<Pair<Request, Error>>>): Fetch`
+8. Performance improvements to the Parallel downloader mode.
+
+
 Version 3.0.7/ Android x version 3.1.1
 Added new pauseAll method on Fetch.
 Added new resumeAll method on Fetch.

README.md

@@ -1,6 +1,6 @@
 
 [![Build Status](https://travis-ci.org/tonyofrancis/Fetch.svg?branch=v2)](https://travis-ci.org/tonyofrancis/Fetch)
-[ ![Download](https://api.bintray.com/packages/tonyofrancis/maven/fetch2/images/download.svg?version=3.0.7) ](https://bintray.com/tonyofrancis/maven/fetch2/3.0.7/link)
+[ ![Download](https://api.bintray.com/packages/tonyofrancis/maven/fetch2/images/download.svg?version=3.0.8) ](https://bintray.com/tonyofrancis/maven/fetch2/3.0.8/link)
 [![Android Arsenal](https://img.shields.io/badge/Android%20Arsenal-Android%20Networking-blue.svg?style=flat)](https://android-arsenal.com/details/1/5196)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/tonyofrancis/Fetch/blob/master/LICENSE)
 
@@ -50,11 +50,11 @@ How to use Fetch
 Using Fetch is easy! Just add the Gradle dependency to your application's build.gradle file.
 
 ```java
-implementation "com.tonyodev.fetch2:fetch2:3.0.7"
+implementation "com.tonyodev.fetch2:fetch2:3.0.8"
 ```
 Androidx use:
 ```java
-implementation "androidx.tonyodev.fetch2:xfetch2:3.1.1"
+implementation "androidx.tonyodev.fetch2:xfetch2:3.1.2"
 ```
 
 Next, get an instance of Fetch and request a download.
@@ -220,11 +220,11 @@ to use the OkHttp Downloader instead. You can create your own custom downloaders
 if necessary. See the Java docs for details.
 
 ```java
-implementation "com.tonyodev.fetch2okhttp:fetch2okhttp:3.0.7"
+implementation "com.tonyodev.fetch2okhttp:fetch2okhttp:3.0.8"
 ```
 Androidx use:
 ```java
-implementation "androidx.tonyodev.fetch2okhttp:xfetch2okhttp:3.1.1"
+implementation "androidx.tonyodev.fetch2okhttp:xfetch2okhttp:3.1.2"
 ```
 
 Set the OkHttp Downloader for Fetch to use.
@@ -246,11 +246,11 @@ If you would like to take advantage of RxJava2 features when using Fetch,
 add the following gradle dependency to your application's build.gradle file.
 
 ```java
-implementation "com.tonyodev.fetch2rx:fetch2rx:3.0.7"
+implementation "com.tonyodev.fetch2rx:fetch2rx:3.0.8"
 ```
 Androidx use:
 ```java
-implementation "androidx.tonyodev.fetch2rx:xfetch2rx:3.1.1"
+implementation "androidx.tonyodev.fetch2rx:xfetch2rx:3.1.2"
 ```
 
 RxFetch makes it super easy to enqueue download requests and query downloads using rxJava2 functional methods.
@@ -286,11 +286,11 @@ added in the coming days.
 
 Start using FetchFileServer by adding the gradle dependency to your application's build.gradle file.
 ```java
-implementation "com.tonyodev.fetch2fileserver:fetch2fileserver:3.0.7"
+implementation "com.tonyodev.fetch2fileserver:fetch2fileserver:3.0.8"
 ```
 Androidx use: 
 ```java
-implementation "androidx.tonyodev.fetch2fileserver:xfetch2fileserver:3.1.1"
+implementation "androidx.tonyodev.fetch2fileserver:xfetch2fileserver:3.1.2"
 ```
 
 Start a FetchFileServer instance and add resource files that it can server to connected clients.
@@ -399,11 +399,11 @@ Fetch1 Migration
 
 Migrate downloads from Fetch1 to Fetch2 using the migration assistant. Add the following gradle dependency to your application's build.gradle file.
 ```java
-implementation "com.tonyodev.fetchmigrator:fetchmigrator:3.0.7"
+implementation "com.tonyodev.fetchmigrator:fetchmigrator:3.0.8"
 ```
 Androidx use:
 ```java
-implementation "androidx.tonyodev.fetchmigrator:xfetchmigrator:3.1.1"
+implementation "androidx.tonyodev.fetchmigrator:xfetchmigrator:3.1.2"
 ```
 
 Then run the Migrator.

fetch2/src/androidTest/java/com/tonyodev/fetch2/DownloadManagerInstrumentedTest.java

@@ -78,7 +78,7 @@ public void useAppContext() throws Exception {
                 progessInterval, fetchLogger, networkInfoProvider, retryOnNetworkGain,
                 downloadInfoUpdater, downloadManagerCoordinator,
                 listenerCoordinator, serverDownloader, false, storageResolver,
-                appContext, namespace, groupInfoProvider, FetchDefaults.DEFAULT_GLOBAL_AUTO_RETRY_ATTEMPTS);
+                appContext, namespace, groupInfoProvider, FetchDefaults.DEFAULT_GLOBAL_AUTO_RETRY_ATTEMPTS, false);
     }
 
     @After

fetch2/src/androidTest/java/com/tonyodev/fetch2/DownloadPriorityIteratorProcessorTest.java

@@ -75,7 +75,8 @@ public void useAppContext() throws Exception {
         final DownloadManager downloadManager = new DownloadManagerImpl(client, concurrentLimit,
                 progessInterval, fetchLogger, networkInfoProvider, retryOnNetworkGain,
                  downloadInfoUpdater, downloadManagerCoordinator,
-                listenerCoordinator, serverDownloader, false, storageResolver, appContext, namespace, groupInfoProvider, FetchDefaults.DEFAULT_GLOBAL_AUTO_RETRY_ATTEMPTS);
+                listenerCoordinator, serverDownloader, false, storageResolver,
+                appContext, namespace, groupInfoProvider, FetchDefaults.DEFAULT_GLOBAL_AUTO_RETRY_ATTEMPTS, false);
         priorityListProcessorImpl = new PriorityListProcessorImpl(
                 new HandlerWrapper(namespace, null),
                 new DownloadProvider(databaseManagerWrapper),

fetch2/src/androidTest/java/com/tonyodev/fetch2/FetchHandlerInstrumentedTest.java

@@ -92,7 +92,8 @@ public void useAppContext() throws Exception {
         final DownloadManager downloadManager = new DownloadManagerImpl(client, concurrentLimit,
                 progessInterval, fetchLogger, networkInfoProvider, retryOnNetworkGain,
                 downloadInfoUpdater, downloadManagerCoordinator,
-                listenerCoordinator, serverDownloader, false, storageResolver, appContext, namespace, groupInfoProvider, FetchDefaults.DEFAULT_GLOBAL_AUTO_RETRY_ATTEMPTS);
+                listenerCoordinator, serverDownloader, false, storageResolver,
+                appContext, namespace, groupInfoProvider, FetchDefaults.DEFAULT_GLOBAL_AUTO_RETRY_ATTEMPTS, false);
         priorityListProcessorImpl = new PriorityListProcessorImpl(
                 handlerWrapper,
                 new DownloadProvider(databaseManagerWrapper),

fetch2/src/main/java/com/tonyodev/fetch2/Fetch.kt

@@ -800,6 +800,22 @@ interface Fetch {
      * */
     fun getContentLengthForRequest(request: Request, fromServer: Boolean, func: Func<Long>, func2: Func<Error>?): Fetch
 
+    /**
+     * Gets the content Length for each request in the passed in list. If the request or contentLength cannot be found in
+     * the Fetch database(meaning Fetch never processed the request and started downloading it) -1 is returned.
+     * However, setting fromServer to true will create a new connection to the server to get the connectLength
+     * if Fetch does not already contain the data in the database for the request.
+     * @param requests Request list. Can be a managed or un-managed list of requests. The requests are not stored in
+     * the fetch database.
+     * @param fromServer If true, fetch will attempt to get the ContentLength
+     * from the server directly by making a network request. Otherwise no action is taken.
+     * @param func callback which returns a list of all the success request pairs with their content length.
+     * @param func2 callback used to return a list of all the request that error when trying to get their content length.
+     * @throws FetchException if this instance of Fetch has been closed.
+     * @return Instance
+     * */
+    fun getContentLengthForRequests(requests:List<Request>, fromServer: Boolean, func: Func<List<Pair<Request,Long>>>, func2: Func<List<Pair<Request, Error>>>): Fetch
+
     /**
      * Gets the Server Response for the url and associated headers.
      * @param url the url. Cannot be null.

fetch2/src/main/java/com/tonyodev/fetch2/downloader/ParallelFileDownloaderImpl.kt

@@ -393,6 +393,11 @@ class ParallelFileDownloaderImpl(private val initialDownload: Download,
             if (downloadSpeedCheckTimeElapsed) {
                 downloadSpeedStartTime = System.nanoTime()
             }
+            try {
+                Thread.sleep(progressReportingIntervalMillis)
+            } catch (e: InterruptedException) {
+                logger.e("FileDownloader", e)
+            }
         }
     }
 
@@ -421,7 +426,7 @@ class ParallelFileDownloaderImpl(private val initialDownload: Download,
                     downloadBlock.downloadedBytes = fileSlice.downloaded
                     downloadBlock.startByte = fileSlice.startBytes
                     downloadBlock.endByte = fileSlice.endBytes
-                    val downloadRequest = getRequestForDownload(downloadInfo, fileSlice.startBytes + fileSlice.downloaded)
+                    val downloadRequest = getRequestForDownload(download = downloadInfo, rangeStart = fileSlice.startBytes + fileSlice.downloaded, segment = fileSlice.position + 1)
                     var downloadResponse: Downloader.Response? = null
                     var saveRandomAccessFile: RandomAccessFile? = null
                     try {

fetch2/src/main/java/com/tonyodev/fetch2/downloader/SequentialFileDownloaderImpl.kt

@@ -342,7 +342,8 @@ class SequentialFileDownloaderImpl(private val initialDownload: Download,
                 requestMethod = GET_REQUEST_METHOD,
                 extras = initialDownload.extras,
                 redirected = false,
-                redirectUrl = "")
+                redirectUrl = "",
+                segment = 1)
     }
 
     private fun getAverageDownloadedBytesPerSecond(): Long {

fetch2/src/main/java/com/tonyodev/fetch2/fetch/FetchImpl.kt

@@ -1063,6 +1063,34 @@ open class FetchImpl constructor(override val namespace: String,
         }
     }
 
+    override fun getContentLengthForRequests(requests: List<Request>, fromServer: Boolean, func: Func<List<Pair<Request, Long>>>, func2: Func<List<Pair<Request, Error>>>): Fetch {
+        synchronized(lock) {
+            throwExceptionIfClosed()
+            handlerWrapper.executeWorkerTask {
+                val results = mutableListOf<Pair<Request, Long>>()
+                val results2 = mutableListOf<Pair<Request, Error>>()
+                for (request in requests) {
+                    try {
+                        results.add(Pair(request, fetchHandler.getContentLengthForRequest(request, fromServer)))
+                    } catch (e: Exception) {
+                        logger.e("Fetch with namespace $namespace error", e)
+                        val error = getErrorFromMessage(e.message)
+                        error.throwable = e
+                        results2.add(Pair(request, error))
+                    }
+                }
+                uiHandler.post {
+                    func.call(results)
+                }
+                uiHandler.post {
+                    func2.call(results2)
+                }
+            }
+            return this
+        }
+    }
+
+
     override fun getServerResponse(url: String,
                                    headers: Map<String, String>?,
                                    func: Func<Downloader.Response>,

fetch2/src/main/java/com/tonyodev/fetch2/util/FetchUtils.kt

@@ -55,7 +55,8 @@ fun getRequestForDownload(download: Download, requestMethod: String = GET_REQUES
 fun getRequestForDownload(download: Download,
                           rangeStart: Long = -1,
                           rangeEnd: Long = -1,
-                          requestMethod: String = GET_REQUEST_METHOD): Downloader.ServerRequest {
+                          requestMethod: String = GET_REQUEST_METHOD,
+                          segment: Int = 1): Downloader.ServerRequest {
     val start = if (rangeStart == -1L) 0 else rangeStart
     val end = if (rangeEnd == -1L) "" else rangeEnd.toString()
     val headers = download.headers.toMutableMap()
@@ -71,7 +72,8 @@ fun getRequestForDownload(download: Download,
             requestMethod = requestMethod,
             extras = download.extras,
             redirected = false,
-            redirectUrl = "")
+            redirectUrl = "",
+            segment = segment)
 }
 
 fun getServerRequestFromRequest(request: Request): Downloader.ServerRequest {
@@ -86,7 +88,8 @@ fun getServerRequestFromRequest(request: Request): Downloader.ServerRequest {
             fileUri = getFileUri(request.file),
             extras = request.extras,
             redirected = false,
-            redirectUrl = "")
+            redirectUrl = "",
+            segment = 1)
 }
 
 fun getCatalogServerRequestFromRequest(request: Request): Downloader.ServerRequest {
@@ -106,7 +109,8 @@ fun getCatalogServerRequestFromRequest(request: Request): Downloader.ServerReque
             fileUri = getFileUri(request.file),
             extras = request.extras,
             redirected = false,
-            redirectUrl = "")
+            redirectUrl = "",
+            segment = 1)
 }
 
 fun getPreviousSliceCount(id: Int, fileTempDir: String): Int {

fetch2core/src/main/java/com/tonyodev/fetch2core/Downloader.kt

@@ -172,7 +172,13 @@ interface Downloader<T, R> : Closeable {
             val redirected: Boolean,
 
             /** redirect url*/
-            val redirectUrl: String)
+            val redirectUrl: String,
+
+            /**
+             * If the request will be downloaded using the parallel download and the file slice count is greater the 1. This field will indicate
+             * the request segment/part.
+             * */
+            val segment: Int)
 
     /**
      * A class that contains the server response information used by Fetch

fetch2okhttp/src/main/java/com/tonyodev/fetch2okhttp/OkHttpDownloader.kt

@@ -72,7 +72,8 @@ open class OkHttpDownloader @JvmOverloads constructor(
                 requestMethod = oldRequest.requestMethod,
                 extras = oldRequest.extras,
                 redirected = true,
-                redirectUrl = redirectUrl)
+                redirectUrl = redirectUrl,
+                segment = oldRequest.segment)
     }
 
     override fun execute(request: Downloader.ServerRequest, interruptMonitor: InterruptMonitor): Downloader.Response? {

fetch2rx/src/main/java/com/tonyodev/fetch2rx/RxFetch.kt

@@ -514,6 +514,20 @@ interface RxFetch {
      * */
     fun getContentLengthForRequest(request: Request, fromServer: Boolean): Convertible<Long>
 
+    /**
+     * Gets the content Length for each request in the passed in list. If the request or contentLength cannot be found in
+     * the Fetch database(meaning Fetch never processed the request and started downloading it) -1 is returned.
+     * However, setting fromServer to true will create a new connection to the server to get the connectLength
+     * if Fetch does not already contain the data in the database for the request.
+     * @param requests Request list. Can be a managed or un-managed list of requests. The requests are not stored in
+     * the fetch database.
+     * @param fromServer If true, fetch will attempt to get the ContentLength
+     * from the server directly by making a network request. Otherwise no action is taken.
+     * @throws FetchException if this instance of Fetch has been closed.
+     * @return convertible containing the success and error result list.
+     * */
+    fun getContentLengthForRequests(requests:List<Request>, fromServer: Boolean): Convertible<Pair<List<Pair<Request, Long>>, List<Pair<Request, Error>>>>
+
     /**
      * Gets the Server Response for the url and associated headers.
      * @param url the url. Cannot be null.

fetch2rx/src/main/java/com/tonyodev/fetch2rx/RxFetchImpl.kt

@@ -987,6 +987,32 @@ open class RxFetchImpl(override val namespace: String,
         }
     }
 
+    override fun getContentLengthForRequests(requests: List<Request>, fromServer: Boolean): Convertible<Pair<List<Pair<Request, Long>>, List<Pair<Request, Error>>>> {
+        return synchronized(lock) {
+            throwExceptionIfClosed()
+            Flowable.just(Pair(requests, fromServer))
+                    .subscribeOn(AndroidSchedulers.from(handlerWrapper.getWorkTaskLooper()))
+                    .flatMap {
+                        throwExceptionIfClosed()
+                        val results = mutableListOf<Pair<Request, Long>>()
+                        val results2 = mutableListOf<Pair<Request, Error>>()
+                        for (request in requests) {
+                            try {
+                                results.add(Pair(request, fetchHandler.getContentLengthForRequest(request, fromServer)))
+                            } catch (e: Exception) {
+                                logger.e("RxFetch with namespace $namespace error", e)
+                                val error = getErrorFromMessage(e.message)
+                                error.throwable = e
+                                results2.add(Pair(request, error))
+                            }
+                        }
+                        Flowable.just(Pair(results as List<Pair<Request, Long>>, results2 as List<Pair<Request, Error>>))
+                    }
+                    .observeOn(uiScheduler)
+                    .toConvertible()
+        }
+    }
+
     override fun getServerResponse(url: String, headers: Map<String, String>?): Convertible<Downloader.Response> {
         return synchronized(lock) {
             throwExceptionIfClosed()

versions.gradle

@@ -1,5 +1,5 @@
 ext {
-    kotlin_version = '1.3.31'
+    kotlin_version = '1.3.40'
     okhttp_version = '3.12.3'
     okhttp_url_version = '3.11.0'
     android_support_version = '28.0.0'
@@ -13,10 +13,10 @@ ext {
     library_target_version = 28
     library_build_tools_version = "28.0.3"
     gradle_tools_version = '3.4.1'
-    rxJava2_version = "2.2.8"
+    rxJava2_version = "2.2.10"
     rxAndroid2_version = "2.1.1"
     timber_version = "4.7.1"
     novoda_bintray_version = "0.9"
-    library_version = "3.0.7"
-    library_version_code = 70
+    library_version = "3.0.8"
+    library_version_code = 71
 }