From 94c256b608e3426ea2a6063a7a64ca553df22f6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lera=20Elvo=C3=A9?= Date: Thu, 25 May 2023 21:45:00 +0300 Subject: [PATCH] add releases manager --- Classes/ReleasesManager.gd | 392 +++++++++++++++++++++++++++++++++++++ UI/Main/ReleasesView.tscn | 22 ++- project.godot | 4 +- 3 files changed, 413 insertions(+), 5 deletions(-) create mode 100644 Classes/ReleasesManager.gd diff --git a/Classes/ReleasesManager.gd b/Classes/ReleasesManager.gd new file mode 100644 index 0000000..0a24988 --- /dev/null +++ b/Classes/ReleasesManager.gd @@ -0,0 +1,392 @@ +extends Node + +const API_SLUG := "https://api.github.com/repos/" +const RELEASES_SLUG := "godotengine/godot/releases" +const ASSET_SLUG := "assets/%s" + +const RELEASES_BASE_FOLDER := "user://releases" +const RELEASES_FILE := RELEASES_BASE_FOLDER + "/releases.json" +const TEMP_FOLDER := "user://_tmp/" + +const FETCH_COOLDOWN := 60.0 # seconds + +var releases: Releases +var github_token: String = OS.get_environment("GITHUB_TOKEN") + +var http: HTTPRequest = HTTPRequest.new() +var is_downloading: bool = false + +var threads: Array[Thread] + +signal releases_fetched(releases: Releases) +signal version_downloaded(version: String) +signal export_templates_downloaded(version: String) + + +func _ready() -> void: + DirAccess.make_dir_absolute(TEMP_FOLDER) + DirAccess.make_dir_absolute(RELEASES_BASE_FOLDER) + add_child(http) + releases = load_releases() + + +func fetch_releases(force_local: bool = false) -> void: + if force_local: + releases_fetched.emit(releases) + return + + if is_downloading: + print("downloading something, try again later") + return + + if releases.last_checked_at + FETCH_COOLDOWN > Time.get_unix_time_from_system(): + print("already fetched recently, returning local") + releases_fetched.emit(releases) + return + + var response_func = func(result: int, _response_code: int, _headers: PackedStringArray, body: PackedByteArray): + if result != OK: + return + + releases.clear() + releases.last_checked_at = Time.get_unix_time_from_system() + var d: Array = JSON.parse_string(body.get_string_from_ascii()) + for i in d: + # tag name is version name + var rm := releases.get_or_create_version_metadata(i.tag_name) + + # we already have this one + if rm.binary_github_asset_id != -1: + continue + + var file_name_placeholder := "Godot_v%s.zip" + # only linux for now + var file_name_template: String = "%s_linux.x86_64" % i.tag_name + var file_name_template_x11: String = "%s_x11.64" % i.tag_name + var file_name := file_name_placeholder % file_name_template + var file_name_x11 := file_name_placeholder % file_name_template_x11 + + var export_template: String = "Godot_v%s_export_templates.tpz" % i.tag_name + # go through assets of this release and find our os + for asset in i.assets: + if asset.name == file_name || asset.name == file_name_x11: + rm.binary_github_asset_id = asset.id + rm.binary_github_filename = asset.name + if asset.name == export_template: + rm.export_github_asset_id = asset.id + rm.export_github_filename = asset.name + + # if we still have an empty item, it's too old to use + # (meaning it's not mirrored on github) + # so we remove it + if rm.is_empty(): + releases.releases.erase(i.tag_name) + + releases_fetched.emit() + is_downloading = false + save_releases() + + var headers := [ + "Accept: application/vnd.github+json", + "Authorization: Bearer %s" % github_token, + "X-GitHub-Api-Version: 2022-11-28", + ] + is_downloading = true + http.request(API_SLUG + RELEASES_SLUG, PackedStringArray(headers)) + http.request_completed.connect(response_func, CONNECT_ONE_SHOT) + + +func download_release(version: String) -> void: + if is_downloading: + print("downloading something, try again later") + return + + if !releases.releases.has(version): + print("this version is not in the index yet or does not exist. fetch first") + return + + if (releases.releases[version] as ReleaseMetadata).binary_path != "": + print("already have this version") + return + + var rm: ReleaseMetadata = releases.releases[version] + var response_func = func(_result: int, response_code: int, _headers: PackedStringArray, body: PackedByteArray, filename: String): + if response_code == 200: + # download happens in place. + # we know the file is zip, so save it to temp first before unpacking + print("got it in place") + print("downloading url: ", rm.binary_github_filename) + var zip_filename := TEMP_FOLDER.path_join("%s.zip" % version) + print("making zip: ", zip_filename) + var f := FileAccess.open(zip_filename, FileAccess.WRITE) + if f.get_error() != OK: + print("zip file creation error:", f.get_error()) + return + f.store_buffer(body) + f.close() + + var zip := ZIPReader.new() + var zerr := zip.open(zip_filename) + if zerr != OK: + print("error opening zip file: ", zerr) + return + var res := zip.read_file(filename.trim_suffix(".zip")) + zip.close() +# clean_tmp() + var d := DirAccess.open(RELEASES_BASE_FOLDER) + d.make_dir_recursive(version) + var new_file := RELEASES_BASE_FOLDER.path_join(version) + new_file = new_file.path_join("godot") + f = FileAccess.open(new_file, FileAccess.WRITE) + f.store_buffer(res) + f.close() + rm.binary_path = new_file + version_downloaded.emit(version) + is_downloading = false + save_releases() + + elif response_code == 302: + # not in place + print("tried downloading, but got a 302 which is not currently supported") + + var headers = [ + "Accept: application/octet-stream", + "Authorization: Bearer %s" % github_token, + "X-GitHub-Api-Version: 2022-11-28", + ] + var asset_url = API_SLUG.path_join(RELEASES_SLUG).path_join(ASSET_SLUG) % rm.binary_github_asset_id + is_downloading = true + http.request(asset_url, PackedStringArray(headers)) + http.request_completed.connect(response_func.bind(rm.binary_github_filename), CONNECT_ONE_SHOT) + + +func download_export_templates(version: String) -> void: + if is_downloading: + print("downloading something, try again later") + return + + if !releases.releases.has(version): + print("this version is not in the index yet or does not exist. fetch first") + return + + if (releases.releases[version] as ReleaseMetadata).export_templates_path != "": + print("already have templates for this version") + return + + var rm: ReleaseMetadata = releases.releases[version] + var response_func = func(_result: int, response_code: int, _headers: PackedStringArray, body: PackedByteArray): + if response_code == 200: + print("got it in place") + print("downloading url: ", rm.export_github_filename) + var zip_filename := TEMP_FOLDER.path_join("%s_export.tpz" % version) + print("making zip: ", zip_filename) + var f := FileAccess.open(zip_filename, FileAccess.WRITE) + if f.get_error() != OK: + print("zip file creation error: ", f.get_error()) + return + f.store_buffer(body) + f.close() + + var zip := ZIPReader.new() + var zerr := zip.open(zip_filename) + if zerr != OK: + print("error opening zip file: ", zerr) + return + var files := zip.get_files() + var d := DirAccess.open(RELEASES_BASE_FOLDER) + d.make_dir_recursive(version.path_join("templates")) + var templates_folder := RELEASES_BASE_FOLDER.path_join(version) + for file in files: + var t := Thread.new() + threads.append(t) + t.start(unpack_zip_file.bind(zip_filename, file, templates_folder.path_join(file), t)) + + zip.close() + # don't clean tmp just yet, since there might be other threads + # that are still unpacking from the same zip. + # side note: why do we have to have the zip in filesystem instead + # of memory? this would be much easier... + rm.export_templates_path = ProjectSettings.globalize_path(templates_folder.path_join("templates")) + export_templates_downloaded.emit(version) + is_downloading = false + save_releases() + elif response_code == 302: + # not in place + print("tried downloading, but got a 302 which is not currently supported") + + var headers = [ + "Accept: application/octet-stream", + "Authorization: Bearer %s" % github_token, + "X-GitHub-Api-Version: 2022-11-28", + ] + var asset_url = API_SLUG.path_join(RELEASES_SLUG).path_join(ASSET_SLUG) % rm.export_github_asset_id + is_downloading = true + http.request(asset_url, PackedStringArray(headers)) + http.request_completed.connect(response_func, CONNECT_ONE_SHOT) + + +func unpack_zip_file(zip_file: String, from_file: String, dest_file: String, thread: Thread) -> void: + print("extracting file ", from_file, " to: ", dest_file) + var z := ZIPReader.new() + var f := z.open(zip_file) + if f != OK: + print("error opening zip file: ", zip_file, " from thread ", thread, ", exiting early.") + _clean_threads.call_deferred(thread) + return + + var file := FileAccess.open(dest_file, FileAccess.WRITE) + file.store_buffer(z.read_file(from_file)) + file.close() + z.close() + + _clean_threads.call_deferred(thread) + + +func _clean_threads(thread: Thread) -> void: + threads.erase(thread) + thread.wait_to_finish() + + if threads.is_empty(): + clean_tmp() + + +func save_releases() -> void: + var f := FileAccess.open(RELEASES_FILE, FileAccess.WRITE) + f.store_string(releases.to_json()) + + +func load_releases() -> Releases: + if !FileAccess.file_exists(RELEASES_FILE): + return Releases.new() + + var f := FileAccess.open(RELEASES_FILE, FileAccess.READ) + return Releases.from_json(f.get_as_text()) + + +func _exit_tree() -> void: + for t in threads: + t.wait_to_finish() + + clean_tmp() + + +func get_version_metadata(version: String) -> ReleaseMetadata: + return releases.releases.get(version) + + +func clean_tmp() -> void: + var d := DirAccess.open(TEMP_FOLDER) + if !d: + return + + var to_delete: Array[String] = [] + d.list_dir_begin() + var file_name := d.get_next() + while file_name != "": + to_delete.append(file_name) + file_name = d.get_next() + d.list_dir_end() + + for f in to_delete: + d.remove(f) + + +# helper methods +func is_version_installed(version: String) -> bool: + return get_version_metadata(version).binary_path != "" + + +func is_version_templates_installed(version) -> bool: + return get_version_metadata(version).export_templates_path != "" + + +class ReleaseMetadata: + var binary_path: String + var is_mono: bool + var export_templates_path: String + + var binary_github_asset_id: int = -1 + var binary_github_filename: String + var export_github_asset_id: int = -1 + var export_github_filename: String + + + func is_empty() -> bool: + return \ + binary_github_asset_id == -1 && export_github_asset_id == -1 + + func to_d() -> Dictionary: + return { + "binary_path": binary_path, + "is_mono": is_mono, + "export_templates_path": export_templates_path, + "binary_github_asset_id": binary_github_asset_id, + "export_github_asset_id": export_github_asset_id, + "binary_github_filename": binary_github_filename, + "export_github_filename": export_github_filename, + } + + + static func from_d(d: Dictionary) -> ReleaseMetadata: + var res := ReleaseMetadata.new() + res.binary_path = d.binary_path + res.is_mono = d.is_mono + res.export_templates_path = d.export_templates_path + res.binary_github_asset_id = d.binary_github_asset_id + res.export_github_asset_id = d.export_github_asset_id + res.binary_github_filename = d.binary_github_filename + res.export_github_filename = d.export_github_filename + return res + + +class Releases: + # version[String]: ReleaseMetadata + var releases: Dictionary + var last_checked_at: float + var os: String + + + func clear() -> void: + for r in releases.keys(): + if (releases[r] as ReleaseMetadata).binary_path == "": + releases.erase(r) + + + func get_or_create_version_metadata(version: String) -> ReleaseMetadata: + if releases.has(version): + return releases[version] + + var r := ReleaseMetadata.new() + releases[version] = r + return r + + + func is_empty() -> bool: + return releases.is_empty() + + + func to_json() -> String: + var releases_dict := {} + for i in releases: + releases_dict[i] = (releases[i] as ReleaseMetadata).to_d() + + var d := { + "os": os, + "last_checked_at": last_checked_at, + "releases": releases_dict, + } + + return JSON.stringify(d, "\t", false) + + + static func from_json(json: String) -> Releases: + var d: Dictionary = JSON.parse_string(json) + var releases_dict := {} + for i in d.releases: + releases_dict[i] = ReleaseMetadata.from_d(d.releases[i]) + + var r := Releases.new() + r.releases = releases_dict + r.os = d.os + r.last_checked_at = d.last_checked_at + return r diff --git a/UI/Main/ReleasesView.tscn b/UI/Main/ReleasesView.tscn index f3dadee..8b0762e 100644 --- a/UI/Main/ReleasesView.tscn +++ b/UI/Main/ReleasesView.tscn @@ -12,7 +12,7 @@ script = ExtResource("1_firao") [node name="Label" type="Label" parent="."] layout_mode = 2 -text = "Releases" +text = "Godots" [node name="PanelContainer" type="PanelContainer" parent="."] layout_mode = 2 @@ -26,10 +26,26 @@ unique_name_in_owner = true layout_mode = 2 size_flags_horizontal = 3 -[node name="CenterContainer" type="CenterContainer" parent="."] +[node name="HBoxContainer" type="HBoxContainer" parent="."] layout_mode = 2 +alignment = 1 -[node name="FetchButton" type="Button" parent="CenterContainer"] +[node name="CenterContainer" type="CenterContainer" parent="HBoxContainer"] +layout_mode = 2 +size_flags_horizontal = 3 + +[node name="FetchButton" type="Button" parent="HBoxContainer/CenterContainer"] unique_name_in_owner = true layout_mode = 2 text = "Fetch" + +[node name="HBoxContainer" type="HBoxContainer" parent="HBoxContainer"] +layout_mode = 2 +size_flags_horizontal = 3 +alignment = 2 + +[node name="StatusLabel" type="Label" parent="HBoxContainer/HBoxContainer"] +unique_name_in_owner = true +visible = false +layout_mode = 2 +text = "Downloading v..." diff --git a/project.godot b/project.godot index d7fc997..d2209b9 100644 --- a/project.godot +++ b/project.godot @@ -15,12 +15,12 @@ run/main_scene="res://UI/Main/GroupsView.tscn" config/use_custom_user_dir=true config/custom_user_dir_name="yagvm" config/features=PackedStringArray("4.0", "Forward Plus") -run/low_processor_mode=true config/icon="res://icon.svg" [autoload] -GithubReleasesFetcher="*res://Classes/GithubReleasesFetcher.gd" +DotEnv="*res://Classes/DotEnv.gd" +ReleasesManager="*res://Classes/ReleasesManager.gd" [display]