evaluation, synthesis & plots updates

parent b11b086c
......@@ -4,3 +4,6 @@
[submodule "data/toys/MovingMNIST"]
path = data/toys/MovingMNIST
url = https://github.com/tychovdo/MovingMNIST.git
[submodule "monitor/trajectories"]
path = monitor/trajectories
url = git@github.com:domkirke/trajectories.git
......@@ -742,7 +742,7 @@ def makeAnalysisFiles(curBatch, transformType, options, oldRoot, newRoot, transf
print("* Computing transforms ...")
for i, target_file in enumerate(curAnalysisFiles):
name = re.sub(os.path.abspath(newRoot), '', target_file)
transform_hash[name+'.dat'] = computeTransform([audioList[i]], transformType, options, out=curAnalysisFiles[i])
_, transform_hash[name+'.dat'] = computeTransform([audioList[i]], transformType, options, out=curAnalysisFiles[i])
return transform_hash
......
......@@ -529,7 +529,7 @@ def computeTransform(audioList, transformType, options={}, out=None):
# else:
# print(out);
# np.savez_compressed(out, currentTransform)
return {'shape':currentTransform.shape, 'strides':currentTransform.strides, 'dtype':currentTransform.dtype}
return currentTransform, {'shape':currentTransform.shape, 'strides':currentTransform.strides, 'dtype':currentTransform.dtype}
def save_memmap(arr, path_out):
......
......@@ -16,7 +16,7 @@ from ..data.signal.transforms import computeTransform, inverseTransform
from ..data import Dataset
from ..utils.trajectory import get_random_trajectory, get_interpolation
from ..utils import checklist, check_dir, choices
from .trajectories import Line, Ellipse, Origin, Normal, Uniform
# CORE FUNCTIONS
......@@ -166,8 +166,7 @@ def resynthesize_files(dataset, model, transformOptions=None, transform=None, me
if transform is not None:
if issubclass(type(transform), (list, tuple)):
transform = transform[0]
pdb.set_trace()
current_transform = computeTransform([current_file], transform, transformOptions)
current_transform, _ = computeTransform([current_file], transform, transformOptions)
current_transform = np.array(current_transform)
# ct = np.copy(current_transform)
else:
......@@ -247,30 +246,50 @@ def resynthesize_files(dataset, model, transformOptions=None, transform=None, me
def trajectory2audio(model, traj_types, transformOptions, n_trajectories=1, n_steps=64, iterations=10, out=None,
preprocessing=None, projection=None, **kwargs):
def trajectory2audio(model, traj_types, transformOptions, n_trajectories=1, n_steps=64, target_length=None, iterations=10,
out=None, layers=None, preprocessing=None, projection=None, **kwargs):
# load model
if not os.path.isdir(out+'/trajectories'):
os.makedirs(out+'/trajectories')
paths = []
for traj_type in traj_types:
for l in range(len(model.platent)):
layers = layers or range(len(model.platent))
if projection is not None:
assert len(projection) == len(layers), "%d projections given for %d latent layers"%(len(projection), len(layers))
if target_length:
fs = fs or transformOptions.get('winSize')
assert fs is not None, "if a target length is given, needs fs keyword"
for layer in layers:
for traj_type in traj_types:
# generate trajectories
trajectories = get_random_trajectory(traj_type, model.platent[l]['dim'], n_trajectories, n_steps)
latent_dim = model.platent[layer]['dim']
if target_length is not None:
t_options = {'n_steps': target_length, 'fs':fs}
else:
t_options = {'n_steps':n_steps}
if traj_type=='line':
trajectory = Line(**t_options, dim=latent_dim, origin=Uniform(dim=latent_dim), end=Uniform(dim=latent_dim))
trajectories = [trajectory() for t in range(n_trajectories)]
elif traj_type == 'ellipse':
geom_args = {'radius':[Uniform()()]*2, 'origin':Origin, 'plane':2}
trajectory = Ellipse(**t_options, dim=model.platent[layer]['dim'], **geom_args)
trajectories = [trajectory() for t in range(n_trajectories)]
# forward trajectory
current_proj = projection
if issubclass(type(projection), list):
projection = projection[l]
projection = projection[layer]
for i,t in enumerate(trajectories):
z = path2audio(model, t, transformOptions, n_interp=1, preprocessing=preprocessing, out=out+'/trajectories/%s_%s_%s'%(traj_type, l, i), iterations=iterations, from_layer=l, **kwargs)
z = path2audio(model, t, transformOptions, n_interp=1, preprocessing=preprocessing, out=out+'/trajectories/%s_%s_%s'%(traj_type, layer, i), iterations=iterations, from_layer=layer, **kwargs)
paths.append(z)
return paths
def interpolate_files(dataset, vae, n_files=1, n_interp=10, out=None, preprocessing=None, preprocess=False,
projections=None, transformType=None, window=None, transformOptions=None, predict=False, **kwargs):
for f in range(n_files):
check_dir('%s/interpolations'%out)
#sequence_length = loaded_data['script_args'].sequence
#files_to_morph = random.choices(range(len(dataset.data)), k=2)
files_to_morph = choices(dataset.files, k=2)
......@@ -308,7 +327,7 @@ def interpolate_files(dataset, vae, n_files=1, n_interp=10, out=None, preprocess
if projections:
grid = gridspec.GridSpec(2,6)
steps = np.linspace(0, n_interp-1, 6, dtype=np.int)
fig = plt.figure(figsize=(12, 6))
fig = plt.figure(figsize=(15, 6))
ax = fig.add_subplot(grid[:2, :2], projection='3d')
proj = projections[l].transform(trajs[l].cpu().detach().numpy())
cmap = plt.cm.get_cmap('plasma', n_interp)
......@@ -316,7 +335,6 @@ def interpolate_files(dataset, vae, n_files=1, n_interp=10, out=None, preprocess
if len(proj.shape) == 3:
for i in range(1, n_interp-1):
ax.plot(proj[i,:, 0], proj[i,:, 1], proj[i,:, 2], color=cmap(i), alpha=0.3)
ax.plot(proj[0, :, 0], proj[0, :, 1], proj[0, :, 2], color=cmap(0))
ax.scatter(proj[0, 0, 0], proj[0, 0, 1], proj[0, 0, 2], color=cmap(0), marker='o')
ax.scatter(proj[0, -1, 0], proj[0, -1, 1], proj[0, -1, 2], color=cmap(0), marker='+')
......@@ -329,6 +347,7 @@ def interpolate_files(dataset, vae, n_files=1, n_interp=10, out=None, preprocess
spec = data_outs[l][steps[i*2+j]].squeeze().cpu().detach().numpy()
ax.imshow(spec, aspect="auto")
ax.set_xticks([]); ax.set_yticks([])
plt.title(f'{files_to_morph[0]}\n{files_to_morph[1]}')
else:
raise NotImplementedError
......
......@@ -17,32 +17,37 @@ class Evaluation(nn.Module):
def parse_params(self, **kwargs):
pass
def forward_model(self, model, input, **kwargs):
def forward_model(self, model, input, multi_decode=True, **kwargs):
preprocessing = kwargs.get('preprocessing')
preprocess = kwargs.get('preproccess')
if preprocessing and preprocess:
input = preprocessing(input)
input = model.format_input_data(input)
vae_out = model.forward(input, **kwargs)
vae_out = model.forward(input, multi_decode=multi_decode, **kwargs)
return vae_out
def evaluate(self, outputs):
pass
def __call__(self, model, loader, baselines=[], *args, **kwargs):
def __call__(self, model, loader, baselines=[], multi_decode=True, *args, **kwargs):
evals = []
print('forwarding model...')
with torch.no_grad():
outs = []; xs=[]; ys=[];
baseline_outs = []
for x, y in loader:
vae_out = self.forward_model(model, x, y=y, **kwargs)
vae_out = self.forward_model(model, x, y=y, multi_decode=multi_decode, **kwargs)
outs.append(decudify(vae_out))
xs.append(decudify(x)); ys.append(decudify(y))
outs = merge_dicts(outs); ys= merge_dicts(ys)
if issubclass(type(outs[0]), list):
outs = [merge_dicts([outs[i][j] for i in range(len(outs))]) for j in range(len(outs[0]))]
ys= merge_dicts(ys)
xs = torch.cat(xs, dim=0)
print(' evaluating...')
evaluation_out = self.evaluate(outs, target=xs, y=ys, model=model, **kwargs)
if issubclass(type(outs), list):
evaluation_out = {f'layer_{i}':self.evaluate(outs[i], target=xs, y=ys, model=model, **kwargs) for i in range(len(outs))}
else:
evaluation_out = self.evaluate(outs, target=xs, y=ys, model=model, **kwargs)
evals.append(evaluation_out)
# baselines
......@@ -96,3 +101,10 @@ class EvaluationContainer(Evaluation):
out = {**out, **e.evaluate(outputs, target=target, model=model, **kwargs)}
return out
def print_evaluation(eval, level=0):
if issubclass(type(eval), dict):
return ''.join(['-'*level+' '+k+'\n'+print_evaluation(v, level=level+1) for k, v in eval.items()])
else:
return '-'*level + ' ' + str(eval) + '\n'
......@@ -8,7 +8,7 @@ import pdb
class ReconstructionEvaluation(Evaluation):
def evaluate_reconstruction(self, vae_out, target, input_params, **kwargs):
def evaluate_reconstruction(self, vae_out, target, input_params, classwise=False, **kwargs):
# Evaluate classical reconstruction losses
reconstruction_loss = LogDensity() + MSE() + L1()
reduction = kwargs.get('reduction', 'none'); reconstruction_loss.reduction = reduction
......
Subproject commit a1b95ddf092283c6833a9a73e34a5ab9c396382e
......@@ -44,7 +44,19 @@ class Embedding(object):
def __call__(self, *args, **kwargs):
return self.transform(*args, **kwargs)
class Dummy(Embedding):
def __init__(self, dim):
self.dim = dim
def __call__(self, z):
return z
def __invert__(self, z):
return z
class PCA(decomposition.PCA, Embedding):
invertible = True
......
......@@ -417,7 +417,7 @@ class ShrubVAE(VanillaVAE):
return list(reversed(outs))
def forward(self, x, y=None, options={}, *args, **kwargs):
def forward(self, x, y=None, options={}, multi_decode=False, *args, **kwargs):
# formats input data
logger = GPULogger(verbose=False)
logger('init')
......@@ -426,20 +426,27 @@ class ShrubVAE(VanillaVAE):
# encode
enc_out, true_lengths = self.encode(x, y=y, **kwargs)
logger('data encoded')
dec_out = self.decode([z['out'] for z in enc_out], y=y, shifts=true_lengths, **kwargs)
logger('data decoded')
x_params = dec_out[0]['out_params']
dec_out = denest_dict(dec_out[1:]) if len(dec_out) > 1 else {}
enc_out = denest_dict(enc_out)
out = {'x_params':x_params, "logdets":enc_out.get('logdet'), "z_preflow":enc_out.get('out_preflow'),
'z_params_dec':dec_out.get('out_params'), 'z_dec':dec_out.get('out'),
'z_params_enc':enc_out['out_params'], 'z_enc':enc_out['out']}
if self.requires_recurrent:
out['recurrent_out'] = enc_out['hidden'][-1]
return out
# decode
layers = range(len(self.platent)) if multi_decode else [-1]
outs = []
for l in layers:
current_zs = [z['out'] for z in enc_out[:(l+1)]]
dec_out = self.decode(current_zs, y=y, shifts=true_lengths, from_layer=l, **kwargs)
logger('data decoded')
x_params = dec_out[0]['out_params']
dec_out_tmp = denest_dict(dec_out[1:]) if len(dec_out) > 1 else {}
enc_out_tmp = denest_dict(enc_out)
out = {'x_params':x_params, "z_preflow":enc_out_tmp.get('out_preflow'),
'z_params_dec':dec_out_tmp.get('out_params'), 'z_dec':dec_out_tmp.get('out'),
'z_params_enc':enc_out_tmp['out_params'], 'z_enc':enc_out_tmp['out']}
if self.requires_recurrent:
out['recurrent_out'] = enc_out['hidden'][-1]
outs.append(out)
if not multi_decode:
outs = outs[0]
return outs
def init_optimizer(self, optim_params, init_scheduler=True):
......
......@@ -112,7 +112,7 @@ class VanillaVAE(AbstractVAE):
outs = list(reversed(outs))
return outs
def forward(self, x, y=None, options={}, *args, **kwargs):
def forward(self, x, y=None, options={}, multi_decode=False, *args, **kwargs):
x = self.format_input_data(x, requires_grad=False)
# logger("data formatted")
enc_out = self.encode(x, y=y, *args, **kwargs)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment