Implement audio layers (#1149)

* Initial work on audio layers

* Load ogg audio files

* Fix playback position

* Support mp3 files

* Play audio at the appropriate position when the animation runs, and stop when the pause button is pressed

* Change audio cel textures for the cels where audio is playing

* Fix audio not playing at the appropriate position

* Don't play audio is layer is invisible

* Set the audio layer names to be the imported audio file names

* Import audio from videos

* Export videos with audio

Only works with mp3 for now

* Remove support for ogg audio files as they cannot be saved

At least until I find a way to save them. Wav files will be supported with Godot 4.4

* Fix adding/removing in-between frames breaking the visual indication of audio cels

* Minor code improvements

* Export audio in videos with custom delay

* Support frame delay

* Change the frame where the audio plays at

* Fix crashes when the audio layer has no track

* Remove unneeded cel properties for audio cels

* Pxo loading/saving

* Load audio files from the audio layer properties

* Change the audio driver to Dummy from the Preferences for performance reasons

* Clone audio layers, disable layer merge and FX buttons when an audio layer is selected

* Easily change the playback frame of an audio layer from the right click menu of cel buttons

* Update Translations.pot

* Some code improvements and documentation

* Stop audio from playing when looping, and the audio does not play at the first frame

* Update audio cel buttons when changing the audio of the layer

* Mute audio layer when hiding it mid-play

* Only plays the portion of the sound that corresponds to the specific frame so maybe we should do that as well

When the animation is not running. If it is running, play the sound properly.

* Some code changes to allow for potential negative frames placement for audio

This woud allow audio to be placed in negative frames, which essentially means that audio would start before the first frame. This is not yet supported, however, because I don't know how to make it work with FFMPEG.

Translations/Translations.pot

@@ -1773,6 +1773,10 @@ msgstr ""
 msgid "If enabled, the application window can become transparent. This affects performance, so keep it off if you don't need it."
 msgstr ""
 
+#. An option found in the preferences, under the Performance section.
+msgid "Use dummy audio driver"
+msgstr ""
+
 #. Found in the Preferences, under Drivers. Specifies the renderer/video driver being used.
 msgid "Renderer:"
 msgstr ""
@@ -2203,6 +2207,10 @@ msgstr ""
 msgid "Unlink Cels"
 msgstr ""
 
+#. An option found in the right click menu of an audio cel. If selected, the audio of the audio layer will start playing from this frame.
+msgid "Play audio here"
+msgstr ""
+
 msgid "Properties"
 msgstr ""
 
@@ -2243,6 +2251,9 @@ msgstr ""
 msgid "Tilemap"
 msgstr ""
 
+msgid "Audio"
+msgstr ""
+
 msgid "Layers"
 msgstr ""
 
@@ -2275,6 +2286,11 @@ msgstr ""
 msgid "Add Tilemap Layer"
 msgstr ""
 
+#. One of the options of the create new layer button.
+#: src/UI/Timeline/AnimationTimeline.tscn
+msgid "Add Audio Layer"
+msgstr ""
+
 #: src/UI/Timeline/AnimationTimeline.tscn
 msgid "Remove current layer"
 msgstr ""
@@ -2405,6 +2421,17 @@ msgstr ""
 msgid "Expand/collapse group"
 msgstr ""
 
+#. Refers to the audio file of an audio layer.
+msgid "Audio file:"
+msgstr ""
+
+msgid "Load file"
+msgstr ""
+
+#. An option in the audio layer properties, allows users to play the audio starting from a specific frame.
+msgid "Play at frame:"
+msgstr ""
+
 msgid "Palette"
 msgstr ""
 

assets/graphics/misc/musical_note.png.import

@@ -0,0 +1,34 @@
+[remap]
+
+importer="texture"
+type="CompressedTexture2D"
+uid="uid://dfjd72smxp6ma"
+path="res://.godot/imported/musical_note.png-f1be7cc6341733e6ffe2fa5b650b80c2.ctex"
+metadata={
+"vram_texture": false
+}
+
+[deps]
+
+source_file="res://assets/graphics/misc/musical_note.png"
+dest_files=["res://.godot/imported/musical_note.png-f1be7cc6341733e6ffe2fa5b650b80c2.ctex"]
+
+[params]
+
+compress/mode=0
+compress/high_quality=false
+compress/lossy_quality=0.7
+compress/hdr_compression=1
+compress/normal_map=0
+compress/channel_pack=0
+mipmaps/generate=false
+mipmaps/limit=-1
+roughness/mode=0
+roughness/src_normal=""
+process/fix_alpha_border=true
+process/premult_alpha=false
+process/normal_map_invert_y=false
+process/hdr_as_srgb=false
+process/hdr_clamp_exposure=false
+process/size_limit=0
+detect_3d/compress_to=1

project.godot

@@ -27,10 +27,6 @@ config/windows_native_icon="res://assets/graphics/icons/icon.ico"
 config/ExtensionsAPI_Version=5
 config/Pxo_Version=4
 
-[audio]
-
-driver/driver="Dummy"
-
 [autoload]
 
 Global="*res://src/Autoload/Global.gd"

src/Autoload/Export.gd

@@ -282,7 +282,7 @@ func process_animation(project := Global.current_project) -> void:
 			for cel in frame.cels:
 				var image := Image.new()
 				image.copy_from(cel.get_image())
-				var duration := frame.duration * (1.0 / project.fps)
+				var duration := frame.get_duration_in_seconds(project.fps)
 				processed_images.append(
 					ProcessedImage.new(image, project.frames.find(frame), duration)
 				)
@@ -298,7 +298,7 @@ func process_animation(project := Global.current_project) -> void:
 				image.copy_from(crop)
 			if trim_images:
 				image = image.get_region(image.get_used_rect())
-			var duration := frame.duration * (1.0 / project.fps)
+			var duration := frame.get_duration_in_seconds(project.fps)
 			processed_images.append(ProcessedImage.new(image, project.frames.find(frame), duration))
 
 
@@ -427,7 +427,7 @@ func export_processed_images(
 
 	if is_single_file_format(project):
 		if is_using_ffmpeg(project.file_format):
-			var video_exported := export_video(export_paths)
+			var video_exported := export_video(export_paths, project)
 			if not video_exported:
 				Global.popup_error(
 					tr("Video failed to export. Ensure that FFMPEG is installed correctly.")
@@ -505,8 +505,9 @@ func export_processed_images(
 
 
 ## Uses FFMPEG to export a video
-func export_video(export_paths: PackedStringArray) -> bool:
+func export_video(export_paths: PackedStringArray, project: Project) -> bool:
 	DirAccess.make_dir_absolute(TEMP_PATH)
+	var video_duration := 0
 	var temp_path_real := ProjectSettings.globalize_path(TEMP_PATH)
 	var input_file_path := temp_path_real.path_join("input.txt")
 	var input_file := FileAccess.open(input_file_path, FileAccess.WRITE)
@@ -516,25 +517,80 @@ func export_video(export_paths: PackedStringArray) -> bool:
 		processed_images[i].image.save_png(temp_file_path)
 		input_file.store_line("file '" + temp_file_name + "'")
 		input_file.store_line("duration %s" % processed_images[i].duration)
+		video_duration += processed_images[i].duration
 	input_file.close()
+
+	# ffmpeg -y -f concat -i input.txt output_path
 	var ffmpeg_execute: PackedStringArray = [
 		"-y", "-f", "concat", "-i", input_file_path, export_paths[0]
 	]
-	var output := []
-	var success := OS.execute(Global.ffmpeg_path, ffmpeg_execute, output, true)
-	print(output)
-	var temp_dir := DirAccess.open(TEMP_PATH)
-	for file in temp_dir.get_files():
-		temp_dir.remove(file)
-	DirAccess.remove_absolute(TEMP_PATH)
+	var success := OS.execute(Global.ffmpeg_path, ffmpeg_execute, [], true)
 	if success < 0 or success > 1:
 		var fail_text := """Video failed to export. Make sure you have FFMPEG installed
 			and have set the correct path in the preferences."""
 		Global.popup_error(tr(fail_text))
+		_clear_temp_folder()
 		return false
+	# Find audio layers
+	var ffmpeg_combine_audio: PackedStringArray = ["-y"]
+	var audio_layer_count := 0
+	var max_audio_duration := 0
+	var adelay_string := ""
+	for layer in project.get_all_audio_layers():
+		if layer.audio is AudioStreamMP3:
+			var temp_file_name := str(audio_layer_count + 1).pad_zeros(number_of_digits) + ".mp3"
+			var temp_file_path := temp_path_real.path_join(temp_file_name)
+			var temp_audio_file := FileAccess.open(temp_file_path, FileAccess.WRITE)
+			temp_audio_file.store_buffer(layer.audio.data)
+			ffmpeg_combine_audio.append("-i")
+			ffmpeg_combine_audio.append(temp_file_path)
+			var delay := floori(layer.playback_position * 1000)
+			# [n]adelay=delay_in_ms:all=1[na]
+			adelay_string += (
+				"[%s]adelay=%s:all=1[%sa];" % [audio_layer_count, delay, audio_layer_count]
+			)
+			audio_layer_count += 1
+			if layer.get_audio_length() >= max_audio_duration:
+				max_audio_duration = layer.get_audio_length()
+	if audio_layer_count > 0:
+		# If we have audio layers, merge them all into one file.
+		for i in audio_layer_count:
+			adelay_string += "[%sa]" % i
+		var amix_inputs_string := "amix=inputs=%s[a]" % audio_layer_count
+		var final_filter_string := adelay_string + amix_inputs_string
+		var audio_file_path := temp_path_real.path_join("audio.mp3")
+		ffmpeg_combine_audio.append_array(
+			PackedStringArray(
+				["-filter_complex", final_filter_string, "-map", '"[a]"', audio_file_path]
+			)
+		)
+		# ffmpeg -i input1 -i input2 ... -i inputn -filter_complex amix=inputs=n output_path
+		var combined_audio_success := OS.execute(Global.ffmpeg_path, ffmpeg_combine_audio, [], true)
+		if combined_audio_success == 0 or combined_audio_success == 1:
+			var copied_video := temp_path_real.path_join("video." + export_paths[0].get_extension())
+			# Then mix the audio file with the video.
+			DirAccess.copy_absolute(export_paths[0], copied_video)
+			# ffmpeg -y -i video_file -i input_audio -c:v copy -map 0:v:0 -map 1:a:0 video_file
+			var ffmpeg_final_video: PackedStringArray = [
+				"-y", "-i", copied_video, "-i", audio_file_path
+			]
+			if max_audio_duration > video_duration:
+				ffmpeg_final_video.append("-shortest")
+			ffmpeg_final_video.append_array(
+				["-c:v", "copy", "-map", "0:v:0", "-map", "1:a:0", export_paths[0]]
+			)
+			OS.execute(Global.ffmpeg_path, ffmpeg_final_video, [], true)
+	_clear_temp_folder()
 	return true
 
 
+func _clear_temp_folder() -> void:
+	var temp_dir := DirAccess.open(TEMP_PATH)
+	for file in temp_dir.get_files():
+		temp_dir.remove(file)
+	DirAccess.remove_absolute(TEMP_PATH)
+
+
 func export_animated(args: Dictionary) -> void:
 	var project: Project = args["project"]
 	var exporter: AImgIOBaseExporter = args["exporter"]

src/Autoload/ExtensionsApi.gd

@@ -631,7 +631,7 @@ class ProjectAPI:
 
 	## Returns the current cel.
 	## Cel type can be checked using function [method get_class_name] inside the cel
-	## type can be GroupCel, PixelCel, Cel3D, or BaseCel.
+	## type can be GroupCel, PixelCel, Cel3D, CelTileMap, AudioCel or BaseCel.
 	func get_current_cel() -> BaseCel:
 		return current_project.get_current_cel()
 

src/Autoload/Global.gd

@@ -14,7 +14,7 @@ signal cel_switched  ## Emitted whenever you select a different cel.
 signal project_data_changed(project: Project)  ## Emitted when project data is modified.
 signal font_loaded  ## Emitted when a new font has been loaded, or an old one gets unloaded.
 
-enum LayerTypes { PIXEL, GROUP, THREE_D, TILEMAP }
+enum LayerTypes { PIXEL, GROUP, THREE_D, TILEMAP, AUDIO }
 enum GridTypes { CARTESIAN, ISOMETRIC, ALL }
 ## ## Used to tell whether a color is being taken from the current theme,
 ## or if it is a custom color.
@@ -490,6 +490,11 @@ var window_transparency := false:
 			return
 		window_transparency = value
 		_save_to_override_file()
+var dummy_audio_driver := false:
+	set(value):
+		if value != dummy_audio_driver:
+			dummy_audio_driver = value
+			_save_to_override_file()
 
 ## Found in Preferences. The time (in minutes) after which backup is created (if enabled).
 var autosave_interval := 1.0:
@@ -726,6 +731,7 @@ func _init() -> void:
 	window_transparency = ProjectSettings.get_setting(
 		"display/window/per_pixel_transparency/allowed"
 	)
+	dummy_audio_driver = ProjectSettings.get_setting("audio/driver/driver") == "Dummy"
 
 
 func _ready() -> void:
@@ -1187,3 +1193,6 @@ func _save_to_override_file() -> void:
 	file.store_line("[display]\n")
 	file.store_line("window/subwindows/embed_subwindows=%s" % single_window_mode)
 	file.store_line("window/per_pixel_transparency/allowed=%s" % window_transparency)
+	if dummy_audio_driver:
+		file.store_line("[audio]\n")
+		file.store_line('driver/driver="Dummy"')

src/Autoload/OpenSave.gd

@@ -45,6 +45,8 @@ func handle_loading_file(file: String) -> void:
 			return
 		var file_name: String = file.get_file().get_basename()
 		Global.control.find_child("ShaderEffect").change_shader(shader, file_name)
+	elif file_ext == "mp3":  # Audio file
+		open_audio_file(file)
 
 	else:  # Image files
 		# Attempt to load as APNG.
@@ -185,8 +187,8 @@ func handle_loading_video(file: String) -> bool:
 			project_size.x = temp_image.get_width()
 		if temp_image.get_height() > project_size.y:
 			project_size.y = temp_image.get_height()
-	DirAccess.remove_absolute(Export.TEMP_PATH)
 	if images_to_import.size() == 0 or project_size == Vector2i.ZERO:
+		DirAccess.remove_absolute(Export.TEMP_PATH)
 		return false  # We didn't find any images, return
 	# If we found images, create a new project out of them
 	var new_project := Project.new([], file.get_basename().get_file(), project_size)
@@ -196,6 +198,14 @@ func handle_loading_video(file: String) -> bool:
 	Global.projects.append(new_project)
 	Global.tabs.current_tab = Global.tabs.get_tab_count() - 1
 	Global.canvas.camera_zoom()
+	var output_audio_file := temp_path_real.path_join("audio.mp3")
+	# ffmpeg -y -i input_file -vn audio.mp3
+	var ffmpeg_execute_audio: PackedStringArray = ["-y", "-i", file, "-vn", output_audio_file]
+	OS.execute(Global.ffmpeg_path, ffmpeg_execute_audio, [], true)
+	if FileAccess.file_exists(output_audio_file):
+		open_audio_file(output_audio_file)
+		temp_dir.remove("audio.mp3")
+	DirAccess.remove_absolute(Export.TEMP_PATH)
 	return true
 
 
@@ -438,6 +448,14 @@ func save_pxo_file(
 			zip_packer.start_file(tileset_path.path_join(str(j)))
 			zip_packer.write_file(tile.image.get_data())
 			zip_packer.close_file()
+	var audio_layers := project.get_all_audio_layers()
+	for i in audio_layers.size():
+		var layer := audio_layers[i]
+		var audio_path := "audio/%s" % i
+		if layer.audio is AudioStreamMP3:
+			zip_packer.start_file(audio_path)
+			zip_packer.write_file(layer.audio.data)
+			zip_packer.close_file()
 	zip_packer.close()
 
 	if temp_path != path:
@@ -902,6 +920,23 @@ func set_new_imported_tab(project: Project, path: String) -> void:
 		Global.tabs.delete_tab(prev_project_pos)
 
 
+func open_audio_file(path: String) -> void:
+	var audio_stream: AudioStream
+	var file := FileAccess.open(path, FileAccess.READ)
+	audio_stream = AudioStreamMP3.new()
+	audio_stream.data = file.get_buffer(file.get_length())
+	if not is_instance_valid(audio_stream):
+		return
+	var project := Global.current_project
+	for layer in project.layers:
+		if layer is AudioLayer and not is_instance_valid(layer.audio):
+			layer.audio = audio_stream
+			return
+	var new_layer := AudioLayer.new(project, path.get_basename().get_file())
+	new_layer.audio = audio_stream
+	Global.animation_timeline.add_layer(new_layer, project)
+
+
 func update_autosave() -> void:
 	if not is_instance_valid(autosave_timer):
 		return

src/Autoload/Tools.gd

@@ -800,7 +800,10 @@ func _cel_switched() -> void:
 	var layer: BaseLayer = Global.current_project.layers[Global.current_project.current_layer]
 	var layer_type := layer.get_layer_type()
 	# Do not make any changes when its the same type of layer, or a group layer
-	if layer_type == _curr_layer_type or layer_type == Global.LayerTypes.GROUP:
+	if (
+		layer_type == _curr_layer_type
+		or layer_type in [Global.LayerTypes.GROUP, Global.LayerTypes.AUDIO]
+	):
 		return
 	_show_relevant_tools(layer_type)
 

src/Classes/Cels/AudioCel.gd

@@ -0,0 +1,18 @@
+class_name AudioCel
+extends BaseCel
+## A class for the properties of cels in AudioLayers.
+## The term "cel" comes from "celluloid" (https://en.wikipedia.org/wiki/Cel).
+
+
+func _init(_opacity := 1.0) -> void:
+	opacity = _opacity
+	image_texture = ImageTexture.new()
+
+
+func get_image() -> Image:
+	var image := Global.current_project.new_empty_image()
+	return image
+
+
+func get_class_name() -> String:
+	return "AudioCel"