From efbaa72afb340e7fe26821e92d2768c258e4c243 Mon Sep 17 00:00:00 2001 From: RealStickman Date: Mon, 3 Oct 2022 19:58:45 +0200 Subject: [PATCH 01/17] Initial commit --- music-normalize/main.py | 84 ++++++++++++++++++++++++++++++++++++++++ music-normalize/notes.md | 13 +++++++ 2 files changed, 97 insertions(+) create mode 100755 music-normalize/main.py create mode 100644 music-normalize/notes.md diff --git a/music-normalize/main.py b/music-normalize/main.py new file mode 100755 index 00000000..2cbabf9b --- /dev/null +++ b/music-normalize/main.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 + +# ffmpeg wrapper +from os import sched_get_priority_max +import ffmpy + +# argument parsing +import argparse + +# executing some commands +import subprocess + +""" +parser = argparse.ArgumentParser(description="") + +# Input file +parser.add_argument("-i", "--input-file", required=True, type=str, help="Input file") + +args = parser.parse_args() + +inputfile = args.input_file +""" +""" +ffmpeg -y -i FalKKonE\ -\ 01\ Aria\ \(From\ \"Berserk:\ The\ Golden\ Age\ Arc\"\).flac -pass 1 -filter:a loudnorm=print_format=json -f flac /dev/null + +ffmpeg -i FalKKonE\ -\ 01\ Aria\ \(From\ \"Berserk:\ The\ Golden\ Age\ Arc\"\).flac -pass 2 -filter:a loudnorm=I=-30.0:linear=true:measured_I=-4.52:measured_LRA=1.90:measured_thresh=-14.64 -c:a libopus -b:a 320k test441.opus +ffmpeg -i $FILE -c:v libx264 -b:v 4000k -pass 2 -filter:a loudnorm=linear=true:measured_I=$input_i:measured_LRA=$input_lra:measured_tp=$input_tp:measured_thresh=$input_thresh -c:a aac -b:a 256k $FILE.mkv +""" + +# FIXME +inputfile = ( + '/home/marc/Downloads/FalKKonE - 01 Aria (From "Berserk: The Golden Age Arc").flac' +) +inputfile = "/home/marc/Downloads/test441.opus" + + +def get_format(inputfile): + # get codec format + # https://stackoverflow.com/a/29610897 + # this shows the codecs of all audio streams present in the file, which shouldn't matter unless you have more than one stream + ff = ffmpy.FFprobe( + inputs={inputfile: None}, + global_options=( + "-v", + "quiet", + "-select_streams a", + "-show_entries stream=codec_name", + "-of default=noprint_wrappers=1:nokey=1", + ), + ) + # print(ff.cmd) + proc = subprocess.Popen(ff.cmd, shell=True, stdout=subprocess.PIPE) + # NOTE read output from previous command + # rstrip: remove trailing newline + # decode: convert from binary string to normal string + format: str = ( + proc.stdout.read() # pyright: ignore[reportOptionalMemberAccess] + .rstrip() + .decode("utf8") + ) + # print(format) + return format + + +def loudness_info(inputfile): + # get format from file + # inputformat = get_format(inputfile) + # NOTE format is actually unnecessary here + ff = ffmpy.FFmpeg( + inputs={inputfile: None}, + outputs={"/dev/null": "-pass 1 -filter:a loudnorm=print_format=json -f null"}, + global_options=("-y"), + ) + + print(ff.cmd) + + +def convert(): + # ff = ffmpy.FFmpeg() + pass + + +if __name__ == "__main__": + loudness_info(inputfile) diff --git a/music-normalize/notes.md b/music-normalize/notes.md new file mode 100644 index 00000000..6ac5edaa --- /dev/null +++ b/music-normalize/notes.md @@ -0,0 +1,13 @@ +# Normalize my music with ffmpeg and python + +[Loudnorm Filter](https://ffmpeg.org/ffmpeg-filters.html#loudnorm) + +[Filtering Guide](https://trac.ffmpeg.org/wiki/FilteringGuide) + + +[Two pass loudnorm](https://superuser.com/a/1312885) +`ffmpeg -y -i FalKKonE\ -\ 01\ Aria\ \(From\ \"Berserk:\ The\ Golden\ Age\ Arc\"\).flac -pass 1 -filter:a loudnorm=print_format=json -f flac /dev/null` + +`ffmpeg -i FalKKonE\ -\ 01\ Aria\ \(From\ \"Berserk:\ The\ Golden\ Age\ Arc\"\).flac -pass 2 -filter:a loudnorm=I=-30.0:linear=true:measured_I=-4.52:measured_LRA=1.90:measured_thresh=-14.64 -c:a libopus -b:a 320k test441.opus` + + From c483092b788b3149682b4df1d6013d2ab5237188 Mon Sep 17 00:00:00 2001 From: RealStickman Date: Mon, 3 Oct 2022 21:39:19 +0200 Subject: [PATCH 02/17] Finish loudness_info to return dict --- music-normalize/main.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/music-normalize/main.py b/music-normalize/main.py index 2cbabf9b..5213dd57 100755 --- a/music-normalize/main.py +++ b/music-normalize/main.py @@ -62,7 +62,7 @@ def get_format(inputfile): return format -def loudness_info(inputfile): +def loudness_info(inputfile) -> dict[str, str]: # get format from file # inputformat = get_format(inputfile) # NOTE format is actually unnecessary here @@ -72,7 +72,22 @@ def loudness_info(inputfile): global_options=("-y"), ) - print(ff.cmd) + # print(ff.cmd) + proc = subprocess.Popen( + ff.cmd, shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE + ) + # NOTE get loudness info from subprocess + # rstrip: remove trailing newline + # decode: convert from binary string to utf8 + # splitlines: list of lines (only 12 last ones, length of the output json) + # join: reassembles the list of lines and separates with "\n" + loudness_json: str = "\n".join( + proc.stdout.read().rstrip().decode("utf8").splitlines()[-12:] + ) + # decode json to dict + loudness: dict[str, str] = json.loads(loudness_json) + # print(loudness_json) + return loudness def convert(): @@ -81,4 +96,4 @@ def convert(): if __name__ == "__main__": - loudness_info(inputfile) + loudness = loudness_info(inputfile=inputfile) From 26d22d410ff333f6ea44187d9f960574281154dc Mon Sep 17 00:00:00 2001 From: RealStickman Date: Mon, 3 Oct 2022 21:39:38 +0200 Subject: [PATCH 03/17] Importing json --- music-normalize/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/music-normalize/main.py b/music-normalize/main.py index 5213dd57..cb383198 100755 --- a/music-normalize/main.py +++ b/music-normalize/main.py @@ -10,6 +10,9 @@ import argparse # executing some commands import subprocess +# parsing json output of loudnorm +import json + """ parser = argparse.ArgumentParser(description="") From b6232f28e5561c1a539df98690e4910f731d8e0e Mon Sep 17 00:00:00 2001 From: RealStickman Date: Mon, 3 Oct 2022 21:39:53 +0200 Subject: [PATCH 04/17] Convert function --- music-normalize/main.py | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/music-normalize/main.py b/music-normalize/main.py index cb383198..146f087d 100755 --- a/music-normalize/main.py +++ b/music-normalize/main.py @@ -34,10 +34,11 @@ ffmpeg -i $FILE -c:v libx264 -b:v 4000k -pass 2 -filter:a loudnorm=linear=true:m inputfile = ( '/home/marc/Downloads/FalKKonE - 01 Aria (From "Berserk: The Golden Age Arc").flac' ) -inputfile = "/home/marc/Downloads/test441.opus" +# inputfile = "/home/marc/Downloads/test441.opus" +outputfile = "/home/marc/Downloads/test441_out.opus" -def get_format(inputfile): +def get_format(inputfile) -> str: # get codec format # https://stackoverflow.com/a/29610897 # this shows the codecs of all audio streams present in the file, which shouldn't matter unless you have more than one stream @@ -93,10 +94,34 @@ def loudness_info(inputfile) -> dict[str, str]: return loudness -def convert(): - # ff = ffmpy.FFmpeg() - pass +def convert(inputfile, outputfile, loudness): + ff = ffmpy.FFmpeg( + inputs={inputfile: None}, + outputs={ + outputfile: "-pass 2" + " " + "-filter:a" + " " + "loudnorm=I=-30.0:" + "measured_I={input_i}:" + "measured_LRA={input_lra}:" + "measured_tp={input_tp}:measured_thresh={input_thresh}:" + "print_format=json" + " " + "-c:a libopus" + " " + "-b:a 320k".format( + input_i=loudness["input_i"], + input_lra=loudness["input_lra"], + input_tp=loudness["input_tp"], + input_thresh=loudness["input_thresh"], + ) + }, + ) + print(ff.cmd) + ff.run() if __name__ == "__main__": loudness = loudness_info(inputfile=inputfile) + convert(inputfile=inputfile, outputfile=outputfile, loudness=loudness) From 3732eadf277bf3268e43e67a752538fb1cd6b882 Mon Sep 17 00:00:00 2001 From: RealStickman Date: Mon, 3 Oct 2022 21:40:13 +0200 Subject: [PATCH 05/17] Add gitignore --- music-normalize/.gitignore | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 music-normalize/.gitignore diff --git a/music-normalize/.gitignore b/music-normalize/.gitignore new file mode 100644 index 00000000..050311cf --- /dev/null +++ b/music-normalize/.gitignore @@ -0,0 +1,6 @@ +.DS_Store +.idea +*.log +tmp/ + + From 1ecd83723ab6ab85e6789db58fe6dc6c865644ca Mon Sep 17 00:00:00 2001 From: RealStickman Date: Tue, 4 Oct 2022 18:16:56 +0200 Subject: [PATCH 06/17] Fully functioning version --- music-normalize/main.py | 165 ++++++++++++++++++++++++++++++++++------ 1 file changed, 143 insertions(+), 22 deletions(-) diff --git a/music-normalize/main.py b/music-normalize/main.py index 146f087d..f414947e 100755 --- a/music-normalize/main.py +++ b/music-normalize/main.py @@ -1,41 +1,46 @@ #!/usr/bin/env python3 # ffmpeg wrapper -from os import sched_get_priority_max +import multiprocessing +from os.path import isdir, isfile import ffmpy # argument parsing import argparse +# multiprocessing stuff +from multiprocessing import Pool +from multiprocessing import cpu_count + # executing some commands import subprocess # parsing json output of loudnorm import json +# file/directory handling +import os + +# most recent starttime for program +import time + +from random import randint + """ -parser = argparse.ArgumentParser(description="") -# Input file -parser.add_argument("-i", "--input-file", required=True, type=str, help="Input file") - -args = parser.parse_args() - -inputfile = args.input_file -""" -""" -ffmpeg -y -i FalKKonE\ -\ 01\ Aria\ \(From\ \"Berserk:\ The\ Golden\ Age\ Arc\"\).flac -pass 1 -filter:a loudnorm=print_format=json -f flac /dev/null - -ffmpeg -i FalKKonE\ -\ 01\ Aria\ \(From\ \"Berserk:\ The\ Golden\ Age\ Arc\"\).flac -pass 2 -filter:a loudnorm=I=-30.0:linear=true:measured_I=-4.52:measured_LRA=1.90:measured_thresh=-14.64 -c:a libopus -b:a 320k test441.opus -ffmpeg -i $FILE -c:v libx264 -b:v 4000k -pass 2 -filter:a loudnorm=linear=true:measured_I=$input_i:measured_LRA=$input_lra:measured_tp=$input_tp:measured_thresh=$input_thresh -c:a aac -b:a 256k $FILE.mkv """ # FIXME -inputfile = ( - '/home/marc/Downloads/FalKKonE - 01 Aria (From "Berserk: The Golden Age Arc").flac' -) +# inputfile = ( +# '/home/marc/Downloads/FalKKonE - 01 Aria (From "Berserk: The Golden Age Arc").flac' +# ) # inputfile = "/home/marc/Downloads/test441.opus" -outputfile = "/home/marc/Downloads/test441_out.opus" +# outputfile = "/home/marc/Downloads/test441_out.opus" + +# srcfolder = "/home/marc/Downloads/MusikRaw" +# destfolder = "/home/marc/Downloads/Musik" + +musicfile_extensions = (".flac", ".wav", ".mp3", ".m4a", ".aac", ".opus") def get_format(inputfile) -> str: @@ -45,8 +50,7 @@ def get_format(inputfile) -> str: ff = ffmpy.FFprobe( inputs={inputfile: None}, global_options=( - "-v", - "quiet", + "-v quiet", "-select_streams a", "-show_entries stream=codec_name", "-of default=noprint_wrappers=1:nokey=1", @@ -66,6 +70,25 @@ def get_format(inputfile) -> str: return format +def remove_picture(inputfile): + """ + This function makes sure no image is attached to the audio stream. + An image might cause problems for the later conversion to opus. + + Parameters: + inputfile (str): Path to file + """ + tmpfile = os.path.splitext(inputfile)[0] + ".tmp" + os.path.splitext(inputfile)[1] + ff = ffmpy.FFmpeg( + inputs={inputfile: None}, + outputs={tmpfile: "-vn -c:a copy"}, + global_options=("-v error"), + ) + ff.run() + os.remove(inputfile) + os.rename(tmpfile, inputfile) + + def loudness_info(inputfile) -> dict[str, str]: # get format from file # inputformat = get_format(inputfile) @@ -91,6 +114,8 @@ def loudness_info(inputfile) -> dict[str, str]: # decode json to dict loudness: dict[str, str] = json.loads(loudness_json) # print(loudness_json) + # print(ff.cmd) + print("Measuring loudness of ", os.path.basename(inputfile)) return loudness @@ -117,11 +142,107 @@ def convert(inputfile, outputfile, loudness): input_thresh=loudness["input_thresh"], ) }, + global_options=("-v error"), ) - print(ff.cmd) + # print(ff.cmd) + print("Working on ", os.path.basename(inputfile)) ff.run() -if __name__ == "__main__": +def main(inputfile: str): + """ + Main program loop + + Parameters: + inputfile (str): Path to input file + """ + # set output folder to parent path + "normalized" + outputfolder = os.path.join(os.path.dirname(inputfile), "normalized") + # NOTE create output folder + # because multiple parallel processes are at work here, + # there might be conflicts with one trying to create the directory although it already exists + # this while loop makes sure the directory does exist + # the try/except block ensures the error is caught and (hopefully) doesn't happen again just after with random sleep + # there's very likely a better way to do this, idk + while not os.path.isdir(outputfolder): + try: + os.mkdir(outputfolder) + except: + time.sleep(randint(0, 4)) + + # output file path + noext_infile: str = os.path.splitext(os.path.basename(inputfile))[0] + outputfile: str = os.path.join(outputfolder, noext_infile + ".opus") + + # print(inputfile) + # print(os.path.dirname(inputfile)) + # print(os.path.basename(inputfile)) + # print(outputfile) + + remove_picture(inputfile=inputfile) loudness = loudness_info(inputfile=inputfile) convert(inputfile=inputfile, outputfile=outputfile, loudness=loudness) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="") + + # Input directory + parser.add_argument( + "-i", "--input-dir", required=True, type=str, help="Input source directory" + ) + + # number of cpus/threads to use, defaults to all available + parser.add_argument( + "-c", + "--cpu-count", + required=False, + type=int, + help="Number of cpu cores", + default=multiprocessing.cpu_count(), + ) + + args = parser.parse_args() + + srcfolder = args.input_dir + + cpu = args.cpu_count + + # NOTE DEALING WITH TIME + # so it only runs for the modified files since the last run + # time of this run + starttime = time.time() + # file where last run is stored + timefile = os.path.join(srcfolder, "run.time") + + # get time of previous run + if os.path.isfile(timefile): + with open(timefile, "r") as file: + timeprev = file.read() + else: + timeprev = 0 + + # FIXME + timeprev = 0 + + # print(timeprev) + + musicfiles = [] + for root, dirs, files in os.walk(srcfolder): + # ignore the "normalized" subfolder + dirs[:] = [d for d in dirs if d not in ["normalized"]] + for file in files: + if file.endswith(musicfile_extensions): + filepath = os.path.join(root, file) + # only file newer than the last run are added + if os.path.getmtime(filepath) >= float(timeprev): + musicfiles.append(os.path.join(root, file)) + + # print(musicfiles) + + with Pool(cpu) as p: + p.map(main, musicfiles) + + # write this run's time into file + with open(timefile, "w") as file: + file.write(str(starttime)) From d584ab9bded4a1f89342bd1e2861075dcc739c15 Mon Sep 17 00:00:00 2001 From: RealStickman Date: Tue, 4 Oct 2022 18:40:29 +0200 Subject: [PATCH 07/17] Always overwrite normalized files --- music-normalize/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/music-normalize/main.py b/music-normalize/main.py index f414947e..b9875f47 100755 --- a/music-normalize/main.py +++ b/music-normalize/main.py @@ -142,7 +142,7 @@ def convert(inputfile, outputfile, loudness): input_thresh=loudness["input_thresh"], ) }, - global_options=("-v error"), + global_options=("-y", "-v error"), ) # print(ff.cmd) print("Working on ", os.path.basename(inputfile)) From f26177b6d7cdb005ae95794d2c1ff7c8373547eb Mon Sep 17 00:00:00 2001 From: RealStickman Date: Tue, 4 Oct 2022 18:40:40 +0200 Subject: [PATCH 08/17] Add reset option. Use mtime from the end of the run --- music-normalize/main.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/music-normalize/main.py b/music-normalize/main.py index b9875f47..d2209576 100755 --- a/music-normalize/main.py +++ b/music-normalize/main.py @@ -202,29 +202,35 @@ if __name__ == "__main__": default=multiprocessing.cpu_count(), ) + # in case you wanted to rerun the conversion for everything + parser.add_argument( + "-r", + "--reset", + required=False, + action="store_true", + help="Rerun conversion for all files", + ) + args = parser.parse_args() srcfolder = args.input_dir cpu = args.cpu_count - # NOTE DEALING WITH TIME - # so it only runs for the modified files since the last run - # time of this run - starttime = time.time() - # file where last run is stored + reset = args.reset + + # file where last run timestamp is stored timefile = os.path.join(srcfolder, "run.time") # get time of previous run - if os.path.isfile(timefile): + if reset: + timeprev = 0 + elif os.path.isfile(timefile): with open(timefile, "r") as file: timeprev = file.read() else: timeprev = 0 - # FIXME - timeprev = 0 - # print(timeprev) musicfiles = [] @@ -245,4 +251,4 @@ if __name__ == "__main__": # write this run's time into file with open(timefile, "w") as file: - file.write(str(starttime)) + file.write(str(time.time())) From c14fc5a77db89e95c86fd460c8d0beaf04150411 Mon Sep 17 00:00:00 2001 From: RealStickman Date: Tue, 4 Oct 2022 18:55:10 +0200 Subject: [PATCH 09/17] Move prints to beginning of function --- music-normalize/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/music-normalize/main.py b/music-normalize/main.py index d2209576..7f467533 100755 --- a/music-normalize/main.py +++ b/music-normalize/main.py @@ -90,6 +90,7 @@ def remove_picture(inputfile): def loudness_info(inputfile) -> dict[str, str]: + print("Measuring loudness of ", os.path.basename(inputfile)) # get format from file # inputformat = get_format(inputfile) # NOTE format is actually unnecessary here @@ -115,11 +116,11 @@ def loudness_info(inputfile) -> dict[str, str]: loudness: dict[str, str] = json.loads(loudness_json) # print(loudness_json) # print(ff.cmd) - print("Measuring loudness of ", os.path.basename(inputfile)) return loudness def convert(inputfile, outputfile, loudness): + print("Working on ", os.path.basename(inputfile)) ff = ffmpy.FFmpeg( inputs={inputfile: None}, outputs={ @@ -145,7 +146,6 @@ def convert(inputfile, outputfile, loudness): global_options=("-y", "-v error"), ) # print(ff.cmd) - print("Working on ", os.path.basename(inputfile)) ff.run() From b145688b6bda68d249c7e20a773cd5fdbcab4525 Mon Sep 17 00:00:00 2001 From: RealStickman Date: Sat, 8 Oct 2022 14:15:24 +0200 Subject: [PATCH 10/17] Separated ffmpy arguments into dicts. (use later to include covers) --- music-normalize/main.py | 56 +++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/music-normalize/main.py b/music-normalize/main.py index 7f467533..5e0c646a 100755 --- a/music-normalize/main.py +++ b/music-normalize/main.py @@ -121,31 +121,36 @@ def loudness_info(inputfile) -> dict[str, str]: def convert(inputfile, outputfile, loudness): print("Working on ", os.path.basename(inputfile)) + # coverpath = os.path.join(os.path.dirname(inputfile), "cover.jpg") + # NOTE including covers into ogg/opus containers currently doesn't work + # https://trac.ffmpeg.org/ticket/4448 + inputcmd = {inputfile: None} + outputcmd = { + outputfile: "-pass 2" + " " + "-filter:a" + " " + "loudnorm=I=-30.0:" + "measured_I={input_i}:" + "measured_LRA={input_lra}:" + "measured_tp={input_tp}:measured_thresh={input_thresh}" + " " + "-c:a libopus" + " " + "-b:a 320k".format( + input_i=loudness["input_i"], + input_lra=loudness["input_lra"], + input_tp=loudness["input_tp"], + input_thresh=loudness["input_thresh"], + ) + } + ff = ffmpy.FFmpeg( - inputs={inputfile: None}, - outputs={ - outputfile: "-pass 2" - " " - "-filter:a" - " " - "loudnorm=I=-30.0:" - "measured_I={input_i}:" - "measured_LRA={input_lra}:" - "measured_tp={input_tp}:measured_thresh={input_thresh}:" - "print_format=json" - " " - "-c:a libopus" - " " - "-b:a 320k".format( - input_i=loudness["input_i"], - input_lra=loudness["input_lra"], - input_tp=loudness["input_tp"], - input_thresh=loudness["input_thresh"], - ) - }, + inputs=inputcmd, + outputs=outputcmd, global_options=("-y", "-v error"), ) - # print(ff.cmd) + print(ff.cmd) ff.run() @@ -179,12 +184,15 @@ def main(inputfile: str): # print(os.path.basename(inputfile)) # print(outputfile) - remove_picture(inputfile=inputfile) + # remove_picture(inputfile=inputfile) loudness = loudness_info(inputfile=inputfile) convert(inputfile=inputfile, outputfile=outputfile, loudness=loudness) if __name__ == "__main__": + # start time of program + starttime = time.time() + parser = argparse.ArgumentParser(description="") # Input directory @@ -251,4 +259,4 @@ if __name__ == "__main__": # write this run's time into file with open(timefile, "w") as file: - file.write(str(time.time())) + file.write(str(starttime)) From d5c56f5f44c88df2a4c7bb06982266ed9b3b990d Mon Sep 17 00:00:00 2001 From: RealStickman Date: Sat, 8 Oct 2022 14:15:58 +0200 Subject: [PATCH 11/17] Remove print statement --- music-normalize/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/music-normalize/main.py b/music-normalize/main.py index 5e0c646a..590626aa 100755 --- a/music-normalize/main.py +++ b/music-normalize/main.py @@ -150,7 +150,7 @@ def convert(inputfile, outputfile, loudness): outputs=outputcmd, global_options=("-y", "-v error"), ) - print(ff.cmd) + # print(ff.cmd) ff.run() From ae73c4974214430929d183aa2ac7b429e3ad7823 Mon Sep 17 00:00:00 2001 From: RealStickman Date: Sun, 13 Nov 2022 21:54:24 +0100 Subject: [PATCH 12/17] WIP print dynamically normalized songs --- music-normalize/main.py | 50 ++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/music-normalize/main.py b/music-normalize/main.py index 590626aa..4b0b3eac 100755 --- a/music-normalize/main.py +++ b/music-normalize/main.py @@ -26,6 +26,8 @@ import time from random import randint +from itertools import repeat + """ """ @@ -119,7 +121,7 @@ def loudness_info(inputfile) -> dict[str, str]: return loudness -def convert(inputfile, outputfile, loudness): +def convert(inputfile, outputfile, loudness, nonlinear): print("Working on ", os.path.basename(inputfile)) # coverpath = os.path.join(os.path.dirname(inputfile), "cover.jpg") # NOTE including covers into ogg/opus containers currently doesn't work @@ -131,9 +133,11 @@ def convert(inputfile, outputfile, loudness): "-filter:a" " " "loudnorm=I=-30.0:" + "LRA=5.0:" "measured_I={input_i}:" "measured_LRA={input_lra}:" - "measured_tp={input_tp}:measured_thresh={input_thresh}" + "measured_tp={input_tp}:measured_thresh={input_thresh}:" + "print_format=json" " " "-c:a libopus" " " @@ -148,13 +152,28 @@ def convert(inputfile, outputfile, loudness): ff = ffmpy.FFmpeg( inputs=inputcmd, outputs=outputcmd, - global_options=("-y", "-v error"), + # global_options=("-y", "-v error"), + global_options=("-y"), ) - # print(ff.cmd) - ff.run() + # ff.run() + proc = subprocess.Popen( + ff.cmd, shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE + ) + # NOTE get loudness info from subprocess + # rstrip: remove trailing newline + # decode: convert from binary string to utf8 + # splitlines: list of lines (only 12 last ones, length of the output json) + # join: reassembles the list of lines and separates with "\n" + loudness_json: str = "\n".join( + proc.stdout.read().rstrip().decode("utf8").splitlines()[-12:] + ) + # decode json to dict + loudness_new: dict[str, str] = json.loads(loudness_json) + if loudness_new["normalization_type"] != "linear": + nonlinear.append([inputfile, loudness_new]) -def main(inputfile: str): +def main(inputfile: str, nonlinear: list): """ Main program loop @@ -186,7 +205,15 @@ def main(inputfile: str): # remove_picture(inputfile=inputfile) loudness = loudness_info(inputfile=inputfile) - convert(inputfile=inputfile, outputfile=outputfile, loudness=loudness) + convert( + inputfile=inputfile, + outputfile=outputfile, + loudness=loudness, + nonlinear=nonlinear, + ) + + # FIXME the dictionary works here + print(nonlinear) if __name__ == "__main__": @@ -230,6 +257,9 @@ if __name__ == "__main__": # file where last run timestamp is stored timefile = os.path.join(srcfolder, "run.time") + # list of non-linear normalizations + nonlinear: list[str] = [] + # get time of previous run if reset: timeprev = 0 @@ -255,8 +285,12 @@ if __name__ == "__main__": # print(musicfiles) with Pool(cpu) as p: - p.map(main, musicfiles) + p.starmap(main, zip(musicfiles, repeat(nonlinear))) # write this run's time into file with open(timefile, "w") as file: file.write(str(starttime)) + + # FIXME empty dictionary here + print("Dynamically normalized music:") + print(nonlinear) From 294d7edaf7becb2609fa3eed053ea53aad782ec6 Mon Sep 17 00:00:00 2001 From: RealStickman Date: Mon, 14 Nov 2022 16:02:02 +0100 Subject: [PATCH 13/17] Add small sample pool function --- music-normalize/main.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/music-normalize/main.py b/music-normalize/main.py index 4b0b3eac..ba9e6765 100755 --- a/music-normalize/main.py +++ b/music-normalize/main.py @@ -294,3 +294,19 @@ if __name__ == "__main__": # FIXME empty dictionary here print("Dynamically normalized music:") print(nonlinear) + +""" +from multiprocessing.pool import ThreadPool + +def test(i): + return([i,{"2":i*2,"3":i*3}]) + + +list = [1,2,3,4,5] + +with ThreadPool(8) as t: + out = t.starmap(test,zip(list)) + +for i in out: + print(i) +""" From 1a2631557842ec4f727e8ca3f660210e0ec0559c Mon Sep 17 00:00:00 2001 From: RealStickman Date: Mon, 14 Nov 2022 16:04:25 +0100 Subject: [PATCH 14/17] Same example with process pool --- music-normalize/main.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/music-normalize/main.py b/music-normalize/main.py index ba9e6765..04796e20 100755 --- a/music-normalize/main.py +++ b/music-normalize/main.py @@ -309,4 +309,20 @@ with ThreadPool(8) as t: for i in out: print(i) + + +from multiprocessing import Pool + +def test(i): + return([i,{"2":i*2,"3":i*3}]) + +if __name__ == "__main__": + list = [1,2,3,4,5] + + with Pool(7) as p: + out = p.starmap(test,zip(list)) + + for i in out: + print(i) """ + From 895a19eb78cd00b05a1f1b04538bf4b5ce20eac6 Mon Sep 17 00:00:00 2001 From: RealStickman Date: Mon, 14 Nov 2022 20:15:52 +0100 Subject: [PATCH 15/17] Output dynamically normalised songs at the end --- music-normalize/main.py | 61 ++++++++++------------------------------- 1 file changed, 15 insertions(+), 46 deletions(-) diff --git a/music-normalize/main.py b/music-normalize/main.py index 04796e20..ae6ed3f9 100755 --- a/music-normalize/main.py +++ b/music-normalize/main.py @@ -26,7 +26,7 @@ import time from random import randint -from itertools import repeat +from typing import Any, Optional """ @@ -121,7 +121,7 @@ def loudness_info(inputfile) -> dict[str, str]: return loudness -def convert(inputfile, outputfile, loudness, nonlinear): +def convert(inputfile, outputfile, loudness) -> Optional[list[Any]]: print("Working on ", os.path.basename(inputfile)) # coverpath = os.path.join(os.path.dirname(inputfile), "cover.jpg") # NOTE including covers into ogg/opus containers currently doesn't work @@ -133,7 +133,7 @@ def convert(inputfile, outputfile, loudness, nonlinear): "-filter:a" " " "loudnorm=I=-30.0:" - "LRA=5.0:" + "LRA=9.0:" "measured_I={input_i}:" "measured_LRA={input_lra}:" "measured_tp={input_tp}:measured_thresh={input_thresh}:" @@ -170,10 +170,11 @@ def convert(inputfile, outputfile, loudness, nonlinear): # decode json to dict loudness_new: dict[str, str] = json.loads(loudness_json) if loudness_new["normalization_type"] != "linear": - nonlinear.append([inputfile, loudness_new]) + nonlinear: list[Any] = [inputfile, loudness_new] + return nonlinear -def main(inputfile: str, nonlinear: list): +def main(inputfile: str) -> Optional[list[Any]]: """ Main program loop @@ -205,15 +206,13 @@ def main(inputfile: str, nonlinear: list): # remove_picture(inputfile=inputfile) loudness = loudness_info(inputfile=inputfile) - convert( + nonlinear: Optional[list[Any]] = convert( inputfile=inputfile, outputfile=outputfile, loudness=loudness, - nonlinear=nonlinear, ) - # FIXME the dictionary works here - print(nonlinear) + return nonlinear if __name__ == "__main__": @@ -258,7 +257,7 @@ if __name__ == "__main__": timefile = os.path.join(srcfolder, "run.time") # list of non-linear normalizations - nonlinear: list[str] = [] + nonlinear_all: Optional[list[Any]] = [] # get time of previous run if reset: @@ -271,7 +270,7 @@ if __name__ == "__main__": # print(timeprev) - musicfiles = [] + musicfiles: list[str] = [] for root, dirs, files in os.walk(srcfolder): # ignore the "normalized" subfolder dirs[:] = [d for d in dirs if d not in ["normalized"]] @@ -285,44 +284,14 @@ if __name__ == "__main__": # print(musicfiles) with Pool(cpu) as p: - p.starmap(main, zip(musicfiles, repeat(nonlinear))) + nonlinear_all: Optional[list[Any]] = p.map(main, musicfiles) # write this run's time into file with open(timefile, "w") as file: file.write(str(starttime)) - # FIXME empty dictionary here print("Dynamically normalized music:") - print(nonlinear) - -""" -from multiprocessing.pool import ThreadPool - -def test(i): - return([i,{"2":i*2,"3":i*3}]) - - -list = [1,2,3,4,5] - -with ThreadPool(8) as t: - out = t.starmap(test,zip(list)) - -for i in out: - print(i) - - -from multiprocessing import Pool - -def test(i): - return([i,{"2":i*2,"3":i*3}]) - -if __name__ == "__main__": - list = [1,2,3,4,5] - - with Pool(7) as p: - out = p.starmap(test,zip(list)) - - for i in out: - print(i) -""" - + for i in nonlinear_all: + # NOTE ignore empty and "None" values + if i: + print(i) From 689497136d7a504282a32964ec56b16094557599 Mon Sep 17 00:00:00 2001 From: RealStickman Date: Mon, 14 Nov 2022 20:58:03 +0100 Subject: [PATCH 16/17] Change LRA to 7.0 --- music-normalize/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/music-normalize/main.py b/music-normalize/main.py index ae6ed3f9..c36e3e83 100755 --- a/music-normalize/main.py +++ b/music-normalize/main.py @@ -133,7 +133,7 @@ def convert(inputfile, outputfile, loudness) -> Optional[list[Any]]: "-filter:a" " " "loudnorm=I=-30.0:" - "LRA=9.0:" + "LRA=7.0:" "measured_I={input_i}:" "measured_LRA={input_lra}:" "measured_tp={input_tp}:measured_thresh={input_thresh}:" From bfe103fa53b3ca2b20b0bd9efeb33f3ec1b0df4e Mon Sep 17 00:00:00 2001 From: RealStickman Date: Sun, 21 May 2023 13:58:53 +0200 Subject: [PATCH 17/17] Add requirements --- music-normalize/requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 music-normalize/requirements.txt diff --git a/music-normalize/requirements.txt b/music-normalize/requirements.txt new file mode 100644 index 00000000..cb25c650 --- /dev/null +++ b/music-normalize/requirements.txt @@ -0,0 +1 @@ +ffmpy