Commit 54853752 authored by Pierre Donat-Bouillud's avatar Pierre Donat-Bouillud
Browse files

Additional measures

parent f3adfada
Pipeline #987 passed with stage
in 2 minutes and 23 seconds
......@@ -113,7 +113,6 @@ def get_exec_times(graph):
def execute_graph(graph):
tqdm.write("Executing graph " + graph)
subprocess.run([graph_exec, "-m", "-b", "-c", "10000", graph], check=True)
return get_exec_times(graph)
def compare_audio_files(audiofiles):
audiofiles.sort()# Number 0 is always the non-degraded file
......@@ -127,6 +126,7 @@ def compare_audio_files(audiofiles):
y,sr = quality.load_file(degraded, duration=2)
basename,_ = os.path.splitext(degraded)
qualities[basename] = quality.compare_specto(y_nd, sr_nd, y, sr)
return qualities
def process_graph(graph):
......@@ -147,7 +147,8 @@ def process_graph(graph):
for graph in tqdm(glob.iglob("*.ag")):
tqdm.write(graph)
basename,_ = os.path.splitext(os.path.basename(graph))
costs[basename] = execute_graph(graph)
execute_graph(graph)
costs[basename] = get_exec_times(graph)
# Get resulting audiofiles
audiofiles = glob.glob("*.wav")
qualities = compare_audio_files(audiofiles)
......@@ -162,6 +163,7 @@ class GraphResults:
def __repr__(self):
return "{}: {}, {}".format(self.name, self.costs, self.quality)
def process_all_graphs(nb_nodes, dirname, random=False, from_graphs=False):
"""Process on all weakly connected Dags up to nb_nodes"""
if (not args.no_generation) and (not args.continue_exec):
......@@ -201,8 +203,8 @@ def process_all_graphs(nb_nodes, dirname, random=False, from_graphs=False):
basename,_ = os.path.splitext(graph)
result= GraphResults(basename)
if not args.continue_exec or (doexec and basename != last_basename):
#We need to execute the graph
result.costs = execute_graph(graph)
#We need to execute the graph. Results are retrieved at get_exec_times
execute_graph(graph)
elif not doexec and basename == last_basename:
doexec = True
......@@ -219,6 +221,7 @@ def process_all_graphs(nb_nodes, dirname, random=False, from_graphs=False):
nb_degraded_graphs = len(result_graph)
#result_dict["Name"] = prefix # Already extracted from the costs
result_dict["NbDegradedGraphs"] = nb_degraded_graphs
# We also want to get the following measures:
# - are the worst/best graphs in terms of costs and quality the same in
......@@ -229,15 +232,10 @@ def process_all_graphs(nb_nodes, dirname, random=False, from_graphs=False):
# - DONE how many degraded graphs in average for one graph?
# - how many resamplers have been inserted? Downsamplers? Upsamplers?
# TODO later: try to degrade in same order as heuristics and see if it correlates with the order in quality and in cost
# TODO: case of a source => use a real audio file? Or generate a sin wave? Or just noise? Or don't generate sources here?
# Because for now, sources just output a 0 signal, so we get the same quality for each version and
# it does not give an useful ranking for the measured quality.
# TODO: apply merge operation for resampler (the one that inserts a mixer and then a resampler instead of several resamplers)
# Meaningless to compute rank correlation on a vector of size 1
if nb_degraded_graphs > 0:
#TODO: save results of analysis to be used if need to continue?
# Get audio files
audiofiles = glob.glob(prefix+"*.wav")
qualities = compare_audio_files(audiofiles)
......@@ -250,10 +248,12 @@ def process_all_graphs(nb_nodes, dirname, random=False, from_graphs=False):
#If we've not computed a quality for it, it is the non-degraded graph
result.quality = qualities.get(result.name, 1.0)
costs = {}#Used to save the results
costs_mes=[]
qualities_mes=[]
for result in result_graph:
cost,_ = result.costs
cost = result.costs[0]
costs[result.name] = result.costs
costs_mes.append(cost)
qualities_mes.append(result.quality)
......@@ -261,6 +261,9 @@ def process_all_graphs(nb_nodes, dirname, random=False, from_graphs=False):
csvname = prefix.rsplit("-", maxsplit=1)[0] + "-theo.csv"
qualities_th, costs_th = load_csv(csvname)
# Save the measured results
results_to_csv(prefix + "-exec-report", qualities, costs)
# Correlations
kendalltau = GraphResults(prefix)
kendalltau.costs = stats.kendalltau(costs_mes, costs_th, nan_policy='raise')
......@@ -278,13 +281,15 @@ def process_all_graphs(nb_nodes, dirname, random=False, from_graphs=False):
result_dict["KendallTau"] = kendalltau
# Speed increase? How many graphs are quicker than the non-degraded one
increasing_costs_mes = list(sorted(result_graph, key=lambda res: res.costs))
increasing_costs_mes = list(sorted(result_graph, key=lambda res: res.costs[0]))
quicker = 0
while quicker < len(increasing_costs_mes):
if int(increasing_costs_mes[quicker].name.rsplit("-", maxsplit=1)[1]) == 0:
break
quicker = quicker + 1
result_dict["QuickerMes"] = quicker
result_dict["QuickestMes"] = (graph_number(increasing_costs_mes[0].name), increasing_costs_mes[0].costs[0])
result_dict["SlowestMes"] = (graph_number(increasing_costs_mes[-1].name), increasing_costs_mes[-1].costs[0])
results[prefix] = result_dict
......@@ -325,7 +330,7 @@ def results_to_csv(graphname, qualities, costs):
writer.writerow(result)
def result_all_graphs_to_csv(name, results):
fieldnames = ["Name", "NbDegradedGraphs", "CostKT", "CostPKT", "QualityKT", "QualityPKT", "CostSR", "CostPSR", "QualitySR", "QualityPSR"]
fieldnames = ["Name", "NbDegradedGraphs", "CostKT", "CostPKT", "QualityKT", "QualityPKT", "CostSR", "CostPSR", "QualitySR", "QualityPSR", "QuickerMes", "QuickestMesName", "QuickestMes", "SlowestMesName", "SlowestMes"]
with open(name, "w", newline='') as csvfile:
result={}
writer = csv.DictWriter(csvfile, fieldnames, delimiter='\t')
......@@ -350,6 +355,15 @@ def result_all_graphs_to_csv(name, results):
result["QualitySR"] = spearmanr.quality[0]
result["QualityPSR"] = spearmanr.quality[1]
result["QuickerMes"] = res["QuickerMes"]
result["QuickestMesName"] = res["QuickestMes"][0]
result["QuickestMes"] = res["QuickestMes"][1]
result["SlowestMesName"] = res["SlowestMes"][0]
result["SlowestMes"] = res["SlowestMes"][1]
writer.writerow(result)
def load_correlations(filename):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment