add releases manager

Classes/ReleasesManager.gd

@@ -0,0 +1,392 @@
+extends Node
+
+const API_SLUG := "https://api.github.com/repos/"
+const RELEASES_SLUG := "godotengine/godot/releases"
+const ASSET_SLUG := "assets/%s"
+
+const RELEASES_BASE_FOLDER := "user://releases"
+const RELEASES_FILE := RELEASES_BASE_FOLDER + "/releases.json"
+const TEMP_FOLDER := "user://_tmp/"
+
+const FETCH_COOLDOWN := 60.0 # seconds
+
+var releases: Releases
+var github_token: String = OS.get_environment("GITHUB_TOKEN")
+
+var http: HTTPRequest = HTTPRequest.new()
+var is_downloading: bool = false
+
+var threads: Array[Thread]
+
+signal releases_fetched(releases: Releases)
+signal version_downloaded(version: String)
+signal export_templates_downloaded(version: String)
+
+
+func _ready() -> void:
+	DirAccess.make_dir_absolute(TEMP_FOLDER)
+	DirAccess.make_dir_absolute(RELEASES_BASE_FOLDER)
+	add_child(http)
+	releases = load_releases()
+
+
+func fetch_releases(force_local: bool = false) -> void:
+	if force_local:
+		releases_fetched.emit(releases)
+		return
+
+	if is_downloading:
+		print("downloading something, try again later")
+		return
+
+	if releases.last_checked_at + FETCH_COOLDOWN > Time.get_unix_time_from_system():
+		print("already fetched recently, returning local")
+		releases_fetched.emit(releases)
+		return
+
+	var response_func = func(result: int, _response_code: int, _headers: PackedStringArray, body: PackedByteArray):
+		if result != OK:
+			return
+
+		releases.clear()
+		releases.last_checked_at = Time.get_unix_time_from_system()
+		var d: Array = JSON.parse_string(body.get_string_from_ascii())
+		for i in d:
+			# tag name is version name
+			var rm := releases.get_or_create_version_metadata(i.tag_name)
+
+			# we already have this one
+			if rm.binary_github_asset_id != -1:
+				continue
+
+			var file_name_placeholder := "Godot_v%s.zip"
+			# only linux for now
+			var file_name_template: String = "%s_linux.x86_64" % i.tag_name
+			var file_name_template_x11: String = "%s_x11.64" % i.tag_name
+			var file_name := file_name_placeholder % file_name_template
+			var file_name_x11 := file_name_placeholder % file_name_template_x11
+
+			var export_template: String = "Godot_v%s_export_templates.tpz" % i.tag_name
+			# go through assets of this release and find our os
+			for asset in i.assets:
+				if asset.name == file_name || asset.name == file_name_x11:
+					rm.binary_github_asset_id = asset.id
+					rm.binary_github_filename = asset.name
+				if asset.name == export_template:
+					rm.export_github_asset_id = asset.id
+					rm.export_github_filename = asset.name
+
+			# if we still have an empty item, it's too old to use
+			# (meaning it's not mirrored on github)
+			# so we remove it
+			if rm.is_empty():
+				releases.releases.erase(i.tag_name)
+
+		releases_fetched.emit()
+		is_downloading = false
+		save_releases()
+
+	var headers := [
+		"Accept: application/vnd.github+json",
+		"Authorization: Bearer %s" % github_token,
+		"X-GitHub-Api-Version: 2022-11-28",
+	]
+	is_downloading = true
+	http.request(API_SLUG + RELEASES_SLUG, PackedStringArray(headers))
+	http.request_completed.connect(response_func, CONNECT_ONE_SHOT)
+
+
+func download_release(version: String) -> void:
+	if is_downloading:
+		print("downloading something, try again later")
+		return
+
+	if !releases.releases.has(version):
+		print("this version is not in the index yet or does not exist. fetch first")
+		return
+
+	if (releases.releases[version] as ReleaseMetadata).binary_path != "":
+		print("already have this version")
+		return
+
+	var rm: ReleaseMetadata = releases.releases[version]
+	var response_func = func(_result: int, response_code: int, _headers: PackedStringArray, body: PackedByteArray, filename: String):
+		if response_code == 200:
+			# download happens in place.
+			# we know the file is zip, so save it to temp first before unpacking
+			print("got it in place")
+			print("downloading url: ", rm.binary_github_filename)
+			var zip_filename := TEMP_FOLDER.path_join("%s.zip" % version)
+			print("making zip: ", zip_filename)
+			var f := FileAccess.open(zip_filename, FileAccess.WRITE)
+			if f.get_error() != OK:
+				print("zip file creation error:", f.get_error())
+				return
+			f.store_buffer(body)
+			f.close()
+
+			var zip := ZIPReader.new()
+			var zerr := zip.open(zip_filename)
+			if zerr != OK:
+				print("error opening zip file: ", zerr)
+				return
+			var res := zip.read_file(filename.trim_suffix(".zip"))
+			zip.close()
+#			clean_tmp()
+			var d := DirAccess.open(RELEASES_BASE_FOLDER)
+			d.make_dir_recursive(version)
+			var new_file := RELEASES_BASE_FOLDER.path_join(version)
+			new_file = new_file.path_join("godot")
+			f = FileAccess.open(new_file, FileAccess.WRITE)
+			f.store_buffer(res)
+			f.close()
+			rm.binary_path = new_file
+			version_downloaded.emit(version)
+			is_downloading = false
+			save_releases()
+
+		elif response_code == 302:
+			# not in place
+			print("tried downloading, but got a 302 which is not currently supported")
+
+	var headers = [
+		"Accept: application/octet-stream",
+		"Authorization: Bearer %s" % github_token,
+		"X-GitHub-Api-Version: 2022-11-28",
+	]
+	var asset_url = API_SLUG.path_join(RELEASES_SLUG).path_join(ASSET_SLUG) % rm.binary_github_asset_id
+	is_downloading = true
+	http.request(asset_url, PackedStringArray(headers))
+	http.request_completed.connect(response_func.bind(rm.binary_github_filename), CONNECT_ONE_SHOT)
+
+
+func download_export_templates(version: String) -> void:
+	if is_downloading:
+		print("downloading something, try again later")
+		return
+
+	if !releases.releases.has(version):
+		print("this version is not in the index yet or does not exist. fetch first")
+		return
+
+	if (releases.releases[version] as ReleaseMetadata).export_templates_path != "":
+		print("already have templates for this version")
+		return
+
+	var rm: ReleaseMetadata = releases.releases[version]
+	var response_func = func(_result: int, response_code: int, _headers: PackedStringArray, body: PackedByteArray):
+		if response_code == 200:
+			print("got it in place")
+			print("downloading url: ", rm.export_github_filename)
+			var zip_filename := TEMP_FOLDER.path_join("%s_export.tpz" % version)
+			print("making zip: ", zip_filename)
+			var f := FileAccess.open(zip_filename, FileAccess.WRITE)
+			if f.get_error() != OK:
+				print("zip file creation error: ", f.get_error())
+				return
+			f.store_buffer(body)
+			f.close()
+
+			var zip := ZIPReader.new()
+			var zerr := zip.open(zip_filename)
+			if zerr != OK:
+				print("error opening zip file: ", zerr)
+				return
+			var files := zip.get_files()
+			var d := DirAccess.open(RELEASES_BASE_FOLDER)
+			d.make_dir_recursive(version.path_join("templates"))
+			var templates_folder := RELEASES_BASE_FOLDER.path_join(version)
+			for file in files:
+				var t := Thread.new()
+				threads.append(t)
+				t.start(unpack_zip_file.bind(zip_filename, file, templates_folder.path_join(file), t))
+
+			zip.close()
+			# don't clean tmp just yet, since there might be other threads
+			# that are still unpacking from the same zip.
+			# side note: why do we have to have the zip in filesystem instead
+			# of memory? this would be much easier...
+			rm.export_templates_path = ProjectSettings.globalize_path(templates_folder.path_join("templates"))
+			export_templates_downloaded.emit(version)
+			is_downloading = false
+			save_releases()
+		elif response_code == 302:
+			# not in place
+			print("tried downloading, but got a 302 which is not currently supported")
+
+	var headers = [
+		"Accept: application/octet-stream",
+		"Authorization: Bearer %s" % github_token,
+		"X-GitHub-Api-Version: 2022-11-28",
+	]
+	var asset_url = API_SLUG.path_join(RELEASES_SLUG).path_join(ASSET_SLUG) % rm.export_github_asset_id
+	is_downloading = true
+	http.request(asset_url, PackedStringArray(headers))
+	http.request_completed.connect(response_func, CONNECT_ONE_SHOT)
+
+
+func unpack_zip_file(zip_file: String, from_file: String, dest_file: String, thread: Thread) -> void:
+	print("extracting file ", from_file, " to: ", dest_file)
+	var z := ZIPReader.new()
+	var f := z.open(zip_file)
+	if f != OK:
+		print("error opening zip file: ", zip_file, " from thread ", thread, ", exiting early.")
+		_clean_threads.call_deferred(thread)
+		return
+
+	var file := FileAccess.open(dest_file, FileAccess.WRITE)
+	file.store_buffer(z.read_file(from_file))
+	file.close()
+	z.close()
+
+	_clean_threads.call_deferred(thread)
+
+
+func _clean_threads(thread: Thread) -> void:
+	threads.erase(thread)
+	thread.wait_to_finish()
+
+	if threads.is_empty():
+		clean_tmp()
+
+
+func save_releases() -> void:
+	var f := FileAccess.open(RELEASES_FILE, FileAccess.WRITE)
+	f.store_string(releases.to_json())
+
+
+func load_releases() -> Releases:
+	if !FileAccess.file_exists(RELEASES_FILE):
+		return Releases.new()
+
+	var f := FileAccess.open(RELEASES_FILE, FileAccess.READ)
+	return Releases.from_json(f.get_as_text())
+
+
+func _exit_tree() -> void:
+	for t in threads:
+		t.wait_to_finish()
+
+	clean_tmp()
+
+
+func get_version_metadata(version: String) -> ReleaseMetadata:
+	return releases.releases.get(version)
+
+
+func clean_tmp() -> void:
+	var d := DirAccess.open(TEMP_FOLDER)
+	if !d:
+		return
+
+	var to_delete: Array[String] = []
+	d.list_dir_begin()
+	var file_name := d.get_next()
+	while file_name != "":
+		to_delete.append(file_name)
+		file_name = d.get_next()
+	d.list_dir_end()
+
+	for f in to_delete:
+		d.remove(f)
+
+
+# helper methods
+func is_version_installed(version: String) -> bool:
+	return get_version_metadata(version).binary_path != ""
+
+
+func is_version_templates_installed(version) -> bool:
+	return get_version_metadata(version).export_templates_path != ""
+
+
+class ReleaseMetadata:
+	var binary_path: String
+	var is_mono: bool
+	var export_templates_path: String
+
+	var binary_github_asset_id: int = -1
+	var binary_github_filename: String
+	var export_github_asset_id: int = -1
+	var export_github_filename: String
+
+
+	func is_empty() -> bool:
+		return \
+		binary_github_asset_id == -1 && export_github_asset_id == -1
+
+	func to_d() -> Dictionary:
+		return {
+			"binary_path": binary_path,
+			"is_mono": is_mono,
+			"export_templates_path": export_templates_path,
+			"binary_github_asset_id": binary_github_asset_id,
+			"export_github_asset_id": export_github_asset_id,
+			"binary_github_filename": binary_github_filename,
+			"export_github_filename": export_github_filename,
+		}
+
+
+	static func from_d(d: Dictionary) -> ReleaseMetadata:
+		var res := ReleaseMetadata.new()
+		res.binary_path = d.binary_path
+		res.is_mono = d.is_mono
+		res.export_templates_path = d.export_templates_path
+		res.binary_github_asset_id = d.binary_github_asset_id
+		res.export_github_asset_id = d.export_github_asset_id
+		res.binary_github_filename = d.binary_github_filename
+		res.export_github_filename = d.export_github_filename
+		return res
+
+
+class Releases:
+	# version[String]: ReleaseMetadata
+	var releases: Dictionary
+	var last_checked_at: float
+	var os: String
+
+
+	func clear() -> void:
+		for r in releases.keys():
+			if (releases[r] as ReleaseMetadata).binary_path == "":
+				releases.erase(r)
+
+
+	func get_or_create_version_metadata(version: String) -> ReleaseMetadata:
+		if releases.has(version):
+			return releases[version]
+
+		var r := ReleaseMetadata.new()
+		releases[version] = r
+		return r
+
+
+	func is_empty() -> bool:
+		return releases.is_empty()
+
+
+	func to_json() -> String:
+		var releases_dict := {}
+		for i in releases:
+			releases_dict[i] = (releases[i] as ReleaseMetadata).to_d()
+
+		var d := {
+			"os": os,
+			"last_checked_at": last_checked_at,
+			"releases": releases_dict,
+		}
+
+		return JSON.stringify(d, "\t", false)
+
+
+	static func from_json(json: String) -> Releases:
+		var d: Dictionary = JSON.parse_string(json)
+		var releases_dict := {}
+		for i in d.releases:
+			releases_dict[i] = ReleaseMetadata.from_d(d.releases[i])
+
+		var r := Releases.new()
+		r.releases = releases_dict
+		r.os = d.os
+		r.last_checked_at = d.last_checked_at
+		return r

UI/Main/ReleasesView.tscn

@@ -12,7 +12,7 @@ script = ExtResource("1_firao")
 
 [node name="Label" type="Label" parent="."]
 layout_mode = 2
-text = "Releases"
+text = "Godots"
 
 [node name="PanelContainer" type="PanelContainer" parent="."]
 layout_mode = 2
@@ -26,10 +26,26 @@ unique_name_in_owner = true
 layout_mode = 2
 size_flags_horizontal = 3
 
-[node name="CenterContainer" type="CenterContainer" parent="."]
+[node name="HBoxContainer" type="HBoxContainer" parent="."]
 layout_mode = 2
+alignment = 1
 
-[node name="FetchButton" type="Button" parent="CenterContainer"]
+[node name="CenterContainer" type="CenterContainer" parent="HBoxContainer"]
+layout_mode = 2
+size_flags_horizontal = 3
+
+[node name="FetchButton" type="Button" parent="HBoxContainer/CenterContainer"]
 unique_name_in_owner = true
 layout_mode = 2
 text = "Fetch"
+
+[node name="HBoxContainer" type="HBoxContainer" parent="HBoxContainer"]
+layout_mode = 2
+size_flags_horizontal = 3
+alignment = 2
+
+[node name="StatusLabel" type="Label" parent="HBoxContainer/HBoxContainer"]
+unique_name_in_owner = true
+visible = false
+layout_mode = 2
+text = "Downloading v..."

project.godot

@@ -15,12 +15,12 @@ run/main_scene="res://UI/Main/GroupsView.tscn"
 config/use_custom_user_dir=true
 config/custom_user_dir_name="yagvm"
 config/features=PackedStringArray("4.0", "Forward Plus")
-run/low_processor_mode=true
 config/icon="res://icon.svg"
 
 [autoload]
 
-GithubReleasesFetcher="*res://Classes/GithubReleasesFetcher.gd"
+DotEnv="*res://Classes/DotEnv.gd"
+ReleasesManager="*res://Classes/ReleasesManager.gd"
 
 [display]