Allow GH token usage. Add Logging. Change Download to downloads/<user…

…>/<repo>/<tag>

.gitignore

@@ -39,4 +39,6 @@ bin/
 .vscode/
 
 ### Mac OS ###
-.DS_Store
+.DS_Store
+downloads
+logs

README.md

@@ -1,3 +1,11 @@
 # GitHub Release and Tag asset downloader
 
-Downloads all assets for all releases and tags, including source archives.
+Downloads all assets for all releases and tags, including source archives.
+
+Run like this:
+
+`java -jar .\github-release-asset-downloader-1.0-SNAPSHOT-all.jar <USER> <REPO>`
+
+of with a GitHub Access Token to reduce rate limiting:
+
+`java -jar .\github-release-asset-downloader-1.0-SNAPSHOT-all.jar <USER> <REPO> <GITHUB_TOKEN>`

build.gradle.kts

@@ -1,5 +1,10 @@
+import java.text.SimpleDateFormat
+import java.util.*
+
 plugins {
     kotlin("jvm") version "2.0.0"
+    id("com.gradleup.shadow") version "8.3.0"
+    application
 }
 
 group = "de.griefed"
@@ -11,6 +16,9 @@ repositories {
 
 dependencies {
     implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.17.1")
+    implementation("io.github.microutils:kotlin-logging:3.0.5")
+    implementation("org.apache.logging.log4j:log4j-api-kotlin:1.4.0")
+    implementation("org.apache.logging.log4j:log4j-core:2.23.1")
     testImplementation(kotlin("test"))
 }
 
@@ -19,4 +27,30 @@ tasks.test {
 }
 kotlin {
     jvmToolchain(21)
+}
+
+application {
+    mainClass = "de.griefed.MainKt"
+}
+
+tasks.jar {
+    duplicatesStrategy = DuplicatesStrategy.INCLUDE
+    manifest {
+        attributes(
+            mapOf(
+                "Built-By" to System.getProperty("user.name"),
+                "Build-Timestamp" to SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ").format(Date()),
+                "Created-By" to "Gradle ${gradle.gradleVersion}",
+                "Build-Jdk" to "${System.getProperty("java.version")} (${System.getProperty("java.vendor")} ${
+                    System.getProperty("java.vm.version")
+                })",
+                "Build-OS" to "${System.getProperty("os.name")} ${System.getProperty("os.arch")} ${
+                    System.getProperty("os.version")
+                }",
+                "Implementation-Vendor" to "Griefed",
+                "Implementation-Version" to project.version,
+                "Implementation-Title" to project.name
+            )
+        )
+    }
 }

src/main/kotlin/GitHubRepo.kt

@@ -3,6 +3,7 @@ package de.griefed
 import com.fasterxml.jackson.databind.DeserializationFeature
 import com.fasterxml.jackson.databind.JsonNode
 import com.fasterxml.jackson.databind.ObjectMapper
+import org.apache.logging.log4j.kotlin.cachedLoggerOf
 import java.io.BufferedReader
 import java.io.File
 import java.io.IOException
@@ -14,88 +15,135 @@ import java.net.URL
 class GitHubRepo {
 
     companion object {
+        private val log by lazy { cachedLoggerOf(GitHubRepo::class.java) }
         private val objectMapper: ObjectMapper = ObjectMapper()
 
         init {
             objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
             objectMapper.enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY)
         }
 
-        fun downloadAssets(user: String, repo: String, destination: String) {
-            val assets = getReleaseAssets(user, repo)
+        fun downloadAssets(user: String, repo: String, token: String = "") {
+            val tags = getTags(user, repo, token)
+            val assets = getReleaseAssets(user, repo, tags, token)
             for (asset in assets) {
-                val folder = File(destination, asset.tag())
-                folder.mkdirs()
-                asset.url().downloadToFile(File(folder,asset.name()))
+                val downloadDestination = File("downloads/$user/$repo", asset.tag())
+                downloadDestination.mkdirs()
+                log.info("Downloading ${asset.tag()}/${asset.name()} from ${asset.url()}")
+                var success: Boolean = false
+                do {
+                    try {
+                        success = asset.url().downloadToFile(File(downloadDestination,asset.name()))
+                    } catch(ex: IOException) {
+                        success = evalException(ex)
+                        if (!success) {
+                            log.info("Rate limited or timeout. Waiting 1 minute... ${ex.message}")
+                            Thread.sleep(1000*60)
+                        }
+                    }
+                } while(!success)
             }
         }
 
-        private fun getTags(user: String, repo: String): List<String> {
+        private fun getTags(user: String, repo: String, token: String): List<String> {
             val tags = mutableListOf<String>()
-            val response = objectMapper.readTree(getResponse(URI("https://api.github.com/repos/$user/$repo/git/refs/tags").toURL()))
-            for (tag in response) {
-                tags.add(tag["ref"].asText().replace("refs/tags/", ""))
+            var response : JsonNode? = null
+            var success: Boolean = false
+            do {
+                try {
+                    response = objectMapper.readTree(getResponse(URI("https://api.github.com/repos/$user/$repo/git/refs/tags").toURL(), token))
+                    success = evalResponse(response)
+                    if (!success) {
+                        log.info("Rate limited or timeout. Waiting 1 minute...")
+                        Thread.sleep(1000*60)
+                    }
+                } catch(ex: IOException) {
+                    success = evalException(ex)
+                    if (!success) {
+                        log.info("Rate limited or timeout. Waiting 1 minute... ${ex.message}")
+                        Thread.sleep(1000*60)
+                    }
+                }
+            } while(!success)
+            if (response != null) {
+                for (tag in response) {
+                    tags.add(tag["ref"].asText().replace("refs/tags/", ""))
+                }
             }
             return tags.toList()
         }
 
-        private fun getReleaseAssets(user: String, repo: String) : List<ReleaseAsset> {
-            val tags = getTags(user, repo)
+        private fun getReleaseAssets(user: String, repo: String, tags: List<String>, token: String) : List<ReleaseAsset> {
             val assets = mutableListOf<ReleaseAsset>()
             for (tag in tags) {
-                try {
-                    val release = getRelease(tag, user, repo)
-                    for (asset in release["assets"]) {
-                        assets.add(
-                            ReleaseAsset(
-                                tag,
-                                asset["name"].asText(),
-                                URI(asset["browser_download_url"].asText()).toURL()
+                var success : Boolean = false
+                do {
+                    try {
+                        val release = getRelease(tag, user, repo, token)
+                        for (asset in release["assets"]) {
+                            assets.add(
+                                ReleaseAsset(
+                                    tag,
+                                    asset["name"].asText(),
+                                    URI(asset["browser_download_url"].asText()).toURL()
+                                )
                             )
-                        )
+                        }
+                        success = true
+                    } catch (ex: IOException) {
+                        success = evalException(ex)
+                        if (!success) {
+                            log.info("Rate limited or timeout. Waiting 1 minute... ${ex.message}")
+                            Thread.sleep(1000*60)
+                        }
                     }
-                    assets.add(
-                        ReleaseAsset(
-                            tag,
-                            "source.tar.gz",
-                            URI(release["tarball_url"].asText()).toURL()
-                        )
-                    )
-                    assets.add(
-                        ReleaseAsset(
-                            tag,
-                            "source.zip",
-                            URI(release["tarball_url"].asText()).toURL()
-                        )
+                } while (!success)
+                assets.add(
+                    ReleaseAsset(
+                        tag,
+                        "source.tar.gz",
+                        URI("https://github.com/$user/$repo/archive/refs/tags/$tag/source.tar.gz").toURL()
                     )
-                } catch (_: Exception) {
-                    //No release available, download sources.
-                    assets.add(
-                        ReleaseAsset(
-                            tag,
-                            "source.zip",
-                            URI("https://github.com/$user/$repo/archive/refs/tags/$tag/source.zip").toURL()
-                        )
+                )
+                assets.add(
+                    ReleaseAsset(
+                        tag,
+                        "source.zip",
+                        URI("https://github.com/$user/$repo/archive/refs/tags/$tag/source.zip").toURL()
                     )
-                    assets.add(
-                        ReleaseAsset(
-                            tag,
-                            "source.tar.gz",
-                            URI("https://github.com/$user/$repo/archive/refs/tags/$tag/source.tar.gz").toURL()
-                        )
-                    )
-                }
+                )
             }
             return assets
         }
 
-        private fun getRelease(tag: String, user: String, repo: String) : JsonNode {
-            return objectMapper.readTree(getResponse(URI("https://api.github.com/repos/$user/$repo/releases/tags/$tag").toURL()))
+        private fun getRelease(tag: String, user: String, repo: String, token: String) : JsonNode {
+            var response : JsonNode
+            var success: Boolean = false
+            do {
+                response = objectMapper.readTree(getResponse(URI("https://api.github.com/repos/$user/$repo/releases/tags/$tag").toURL(), token))
+                success = evalResponse(response)
+                if (!success) {
+                    log.info("Rate limited or timeout. Waiting 1 minute...")
+                    Thread.sleep(1000*60)
+                }
+            } while(!success)
+            return response
+        }
+
+        private fun evalException(ex: IOException) : Boolean {
+            return ex.message!!.matches(".*responded with 404".toRegex())
+        }
+
+        private fun evalResponse(response: JsonNode) : Boolean {
+            return !(response.has("message") && response.has("documentation_url"))
         }
 
-        private fun getResponse(requestUrl: URL): String {
+        private fun getResponse(requestUrl: URL, token: String): String {
             val httpURLConnection = requestUrl.openConnection() as HttpURLConnection
             httpURLConnection.requestMethod = "GET"
+            if (token.isNotBlank()) {
+                httpURLConnection.setRequestProperty("Authorization", "Bearer $token")
+            }
             if (httpURLConnection.responseCode != 200) throw IOException("Request for " + requestUrl + " responded with " + httpURLConnection.responseCode)
             val bufferedReader = BufferedReader(
                 InputStreamReader(httpURLConnection.inputStream)

src/main/kotlin/Main.kt

@@ -1,6 +1,10 @@
 package de.griefed
 
 fun main(args: Array<String>) {
-    GitHubRepo.downloadAssets(args[0], args[1], args[2])
+    if (args.size == 3) {
+        GitHubRepo.downloadAssets(args[0], args[1], args[2])
+    } else {
+        GitHubRepo.downloadAssets(args[0], args[1])
+    }
 }
 

src/main/kotlin/WebUtilities.kt

@@ -6,16 +6,12 @@ import java.nio.channels.Channels
 
 fun URL.downloadToFile(file: File) : Boolean {
     file.create()
-    try {
-        openStream().use { url ->
-            Channels.newChannel(url).use { channel ->
-                file.outputStream().use { stream ->
-                    stream.channel.transferFrom(channel, 0, Long.MAX_VALUE)
-                }
+    openStream().use { url ->
+        Channels.newChannel(url).use { channel ->
+            file.outputStream().use { stream ->
+                stream.channel.transferFrom(channel, 0, Long.MAX_VALUE)
             }
         }
-    } catch (ex: IOException) {
-        file.deleteQuietly()
     }
-    return file.isFile
+    return file.isFile && file.length() > 0
 }

src/main/resources/log4j2.component.properties

@@ -0,0 +1,2 @@
+log4j.configurationFile=log4j2.xml
+log4j.formatMsgNoLookups=true

src/main/resources/log4j2.xml

@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration monitorInterval="30">
+    <Properties>
+        <Property name="log-path">logs</Property>
+        <Property name="archive">${log-path}/archive</Property>
+        <Property name="pattern">%d{ISO8601} %5p [%t] (%F:%L) - %m%n</Property>
+        <Property name="console-pattern">%style{%d{ISO8601}}{dim,white} %highlight{%5p} [%style{%t}{bright,blue}] %style{(%F:%L)}{bright,yellow} - %m%n</Property>
+        <Property name="log-level">INFO</Property>
+        <Property name="log-level-gad">INFO</Property>
+        <Property name="charset">UTF-8</Property>
+    </Properties>
+
+    <Appenders>
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout>
+                <Pattern>
+                    ${console-pattern}
+                </Pattern>
+            </PatternLayout>
+        </Console>
+
+        <RollingFile name="ApplicationLogger" fileName="${log-path}/downloader.log" filePattern="${archive}/downloader.log.%i">
+            <PatternLayout>
+                <Charset>
+                    ${charset}
+                </Charset>
+                <Pattern>
+                    ${pattern}
+                </Pattern>
+            </PatternLayout>
+            <Policies>
+                <OnStartupTriggeringPolicy />
+                <SizeBasedTriggeringPolicy size="10 MB" />
+            </Policies>
+            <DefaultRolloverStrategy max="5" />
+        </RollingFile>
+    </Appenders>
+
+    <Loggers>
+        <Root level="ALL">
+            <AppenderRef ref="Console" level="${log-level}"/>
+            <AppenderRef ref="ApplicationLogger" level="${log-level}" />
+        </Root>
+
+        <Logger name="de.griefed" level="ALL" additivity="false">
+            <AppenderRef ref="Console" level="${log-level-gad}"/>
+            <AppenderRef ref="ApplicationLogger" level="${log-level-gad}" />
+        </Logger>
+    </Loggers>
+</Configuration>