feat: init basic pick job worker

unit-picker/src/main/kotlin/cc/unitmesh/pick/Main.kt

@@ -11,7 +11,7 @@ class Picker : CliktCommand() {
 
     override fun run() {
         val config = PickerConfig(url = url)
-        val worker = CodePicker(config)
+        CodePicker(config).execute()
     }
 }
 

unit-picker/src/main/kotlin/cc/unitmesh/pick/picker/CodePicker.kt

@@ -11,7 +11,7 @@ class CodePicker(val config: PickerConfig) {
     private val logger = org.slf4j.LoggerFactory.getLogger(javaClass)
     private val languageService = LanguageService()
 
-    fun run() {
+    fun execute() {
         // 1. check config.url is a valid url or path
         val codeDir = checkoutCode(config.url, config.branch, config.baseDir)
 

unit-picker/src/main/kotlin/cc/unitmesh/pick/picker/DirectoryWalker.kt

@@ -0,0 +1,123 @@
+package cc.unitmesh.pick.picker
+
+import kotlinx.coroutines.*
+import kotlinx.coroutines.channels.Channel
+import org.archguard.scanner.analyser.count.LanguageWorker
+import org.archguard.scanner.analyser.ignore.Gitignore
+import org.archguard.scanner.analyser.ignore.IgnoreMatcher
+import java.io.File
+
+
+class DirectoryJob(
+    val root: String,
+    val path: String,
+    val ignores: List<IgnoreMatcher>
+)
+
+// ".git", ".hg", ".svn"
+val PathDenyList: List<String> = listOf(".git", ".hg", ".svn")
+
+class DirectoryWalker(
+    val output: Channel<PickJob>,
+    val excludes: List<Regex> = listOf()
+) {
+    private var root: String = ""
+    private val ignores: MutableList<IgnoreMatcher> = mutableListOf()
+    private val dirChannels = mutableListOf<Channel<DirectoryJob>>()
+
+    suspend fun start(workdir: String) = coroutineScope {
+        root = workdir
+        val file = File(workdir)
+        if (!file.exists()) throw Exception("failed to open $workdir")
+
+        if (!file.isDirectory) {
+            launch {
+                LanguageWorker.createFileJob(file).let {
+                    output.send(PickJob.from(it))
+                }
+            }
+        } else {
+            createDirJob(workdir, workdir)
+        }
+    }
+
+    fun readDir(path: String): Array<out File>? {
+        val file = File(path)
+        if (!file.exists()) {
+            throw Exception("failed to open $path")
+        }
+        if (!file.isDirectory) {
+            throw Exception("failed to read $path")
+        }
+
+        return file.listFiles()
+    }
+
+    private fun CoroutineScope.createDirJob(root: String, path: String) {
+        val channel = Channel<DirectoryJob>(1)
+        dirChannels.add(channel)
+
+        launch {
+            channel.send(DirectoryJob(root, path, ignores))
+
+            for (dir in channel) {
+                walk(path)
+            }
+
+            channel.close()
+        }
+    }
+
+    private suspend fun walk(workdir: String) = coroutineScope {
+        val dirents = readDir(workdir) ?: return@coroutineScope
+
+        dirents.map {
+            val name = it.name
+
+            if (name == ".gitignore" || it.name == ".ignore") {
+                val path = File(workdir).resolve(name).toString()
+                val file = Gitignore.create(path) ?: return@map
+                ignores.add(file)
+            }
+        }
+
+        dirents.forEach { file ->
+            val name = file.name
+            val path = file.toString()
+            val isDir = file.isDirectory
+
+
+            for (deny in PathDenyList) {
+                if (path.contains(deny)) {
+                    return@forEach
+                }
+            }
+
+            for (exclude in excludes) {
+                if (exclude.matches(path) || exclude.matches(name)) {
+                    return@forEach
+                }
+            }
+
+            for (ignore in ignores) {
+                if (ignore.match(path, isDir)) {
+                    return@forEach
+                }
+            }
+
+            if (!isDir) {
+                LanguageWorker.createFileJob(file).let {
+                    launch {
+                        output.send(PickJob.from(it))
+                    }
+                }
+            } else {
+                createDirJob(root, file.absolutePath)
+            }
+        }
+
+        if(dirChannels.size >= 1) {
+            dirChannels.removeAt(0).close()
+        }
+    }
+}

unit-picker/src/main/kotlin/cc/unitmesh/pick/picker/PickJob.kt

@@ -0,0 +1,55 @@
+package cc.unitmesh.pick.picker
+
+import kotlinx.serialization.Transient
+import org.archguard.scanner.analyser.count.FileJob
+import java.security.MessageDigest
+
+class PickJob(
+    var language: String = "",
+    var possibleLanguages: List<String> = listOf(),
+    var filename: String = "",
+    var extension: String = "",
+    var location: String = "",
+    var symlocation: String = "",
+    var content: ByteArray = byteArrayOf(),
+    var bytes: Long = 0,
+    var lines: Long = 0,
+    var code: Long = 0,
+    var comment: Long = 0,
+    var blank: Long = 0,
+    var complexity: Long = 0,
+    var weightedComplexity: Double = 0.0,
+    // skip serialisation
+    @Transient
+    var hash: MessageDigest = MessageDigest.getInstance("SHA-256"),
+    var binary: Boolean = false,
+    var minified: Boolean = false,
+    var generated: Boolean = false,
+    var endPoint: Int = 0,
+) {
+    companion object {
+        fun from(fileJob: FileJob): PickJob {
+            return PickJob(
+                language = fileJob.language,
+                possibleLanguages = fileJob.possibleLanguages,
+                filename = fileJob.filename,
+                extension = fileJob.extension,
+                location = fileJob.location,
+                symlocation = fileJob.symlocation,
+                content = fileJob.content,
+                bytes = fileJob.bytes,
+                lines = fileJob.lines,
+                code = fileJob.code,
+                comment = fileJob.comment,
+                blank = fileJob.blank,
+                complexity = fileJob.complexity,
+                weightedComplexity = fileJob.weightedComplexity,
+                hash = fileJob.hash,
+                binary = fileJob.binary,
+                minified = fileJob.minified,
+                generated = fileJob.generated,
+                endPoint = fileJob.endPoint,
+            )
+        }
+    }
+}

unit-picker/src/test/kotlin/cc/unitmesh/pick/picker/CodePickerTest.kt

@@ -6,9 +6,8 @@ import kotlin.test.assertEquals
 class CodePickerTest {
     @Test
     fun should_return_unique_path() {
-        // based on: https://www.debuggex.com/r/fFggA8Uc4YYKjl34
         assertEquals(
-            "github.com/unit-mesh/unit-pick",
+            "github.com/unit-mesh/unit-eval",
             CodePicker.gitUrlToPath("https://github.com/unit-mesh/unit-eval")
         )
         assertEquals(
@@ -64,6 +63,6 @@ class CodePickerTest {
             )
         )
 
-        picker.run()
+        picker.execute()
     }
 }