diff --git a/CAE/caeapplygrads.m b/CAE/caeapplygrads.m new file mode 100644 index 0000000..57b788f --- /dev/null +++ b/CAE/caeapplygrads.m @@ -0,0 +1,29 @@ +function cae = caeapplygrads(cae) + cae.sv = 0; + for j = 1 : numel(cae.a) + for i = 1 : numel(cae.i) +% cae.vik{i}{j} = cae.momentum * cae.vik{i}{j} + cae.alpha ./ (cae.sigma + cae.ddik{i}{j}) .* cae.dik{i}{j}; +% cae.vok{i}{j} = cae.momentum * cae.vok{i}{j} + cae.alpha ./ (cae.sigma + cae.ddok{i}{j}) .* cae.dok{i}{j}; + cae.vik{i}{j} = cae.alpha * cae.dik{i}{j}; + cae.vok{i}{j} = cae.alpha * cae.dok{i}{j}; + cae.sv = cae.sv + sum(cae.vik{i}{j}(:) .^ 2); + cae.sv = cae.sv + sum(cae.vok{i}{j}(:) .^ 2); + + cae.ik{i}{j} = cae.ik{i}{j} - cae.vik{i}{j}; + cae.ok{i}{j} = cae.ok{i}{j} - cae.vok{i}{j}; + end +% cae.vb{j} = cae.momentum * cae.vb{j} + cae.alpha / (cae.sigma + cae.ddb{j}) * cae.db{j}; + cae.vb{j} = cae.alpha * cae.db{j}; + cae.sv = cae.sv + sum(cae.vb{j} .^ 2); + + cae.b{j} = cae.b{j} - cae.vb{j}; + end + + for i = 1 : numel(cae.o) +% cae.vc{i} = cae.momentum * cae.vc{i} + cae.alpha / (cae.sigma + cae.ddc{i}) * cae.dc{i}; + cae.vc{i} = cae.alpha * cae.dc{i}; + cae.sv = cae.sv + sum(cae.vc{i} .^ 2); + + cae.c{i} = cae.c{i} - cae.vc{i}; + end +end diff --git a/CAE/caebbp.m b/CAE/caebbp.m new file mode 100644 index 0000000..6eca671 --- /dev/null +++ b/CAE/caebbp.m @@ -0,0 +1,29 @@ +function cae = caebbp(cae) + + %% backprop deltas + for i = 1 : numel(cae.o) + % output delta delta + cae.odd{i} = (cae.o{i} .* (1 - cae.o{i}) .* cae.edgemask) .^ 2; + % delta delta c + cae.ddc{i} = sum(cae.odd{i}(:)) / size(cae.odd{i}, 1); + end + + for j = 1 : numel(cae.a) % calc activation delta deltas + z = 0; + for i = 1 : numel(cae.o) + z = z + convn(cae.odd{i}, flipall(cae.ok{i}{j} .^ 2), 'full'); + end + cae.add{j} = (cae.a{j} .* (1 - cae.a{j})) .^ 2 .* z; + end + + %% calc params delta deltas + ns = size(cae.odd{1}, 1); + for j = 1 : numel(cae.a) + cae.ddb{j} = sum(cae.add{j}(:)) / ns; + for i = 1 : numel(cae.o) + cae.ddok{i}{j} = convn(flipall(cae.a{j} .^ 2), cae.odd{i}, 'valid') / ns; + cae.ddik{i}{j} = convn(cae.add{j}, flipall(cae.i{i} .^ 2), 'valid') / ns; + end + end + +end diff --git a/CAE/caebp.m b/CAE/caebp.m new file mode 100644 index 0000000..bb8e084 --- /dev/null +++ b/CAE/caebp.m @@ -0,0 +1,34 @@ +function cae = caebp(cae, y) + + %% backprop deltas + cae.L = 0; + for i = 1 : numel(cae.o) + % error + cae.e{i} = (cae.o{i} - y{i}) .* cae.edgemask; + % loss function + cae.L = cae.L + 1/2 * sum(cae.e{i}(:) .^2 ) / size(cae.e{i}, 1); + % output delta + cae.od{i} = cae.e{i} .* (cae.o{i} .* (1 - cae.o{i})); + + cae.dc{i} = sum(cae.od{i}(:)) / size(cae.e{i}, 1); + end + + for j = 1 : numel(cae.a) % calc activation deltas + z = 0; + for i = 1 : numel(cae.o) + z = z + convn(cae.od{i}, flipall(cae.ok{i}{j}), 'full'); + end + cae.ad{j} = cae.a{j} .* (1 - cae.a{j}) .* z; + end + + %% calc gradients + ns = size(cae.e{1}, 1); + for j = 1 : numel(cae.a) + cae.db{j} = sum(cae.ad{j}(:)) / ns; + for i = 1 : numel(cae.o) + cae.dok{i}{j} = convn(flipall(cae.a{j}), cae.od{i}, 'valid') / ns; + cae.dik{i}{j} = convn(cae.ad{j}, flipall(cae.i{i}), 'valid') / ns; + end + end + +end diff --git a/CAE/caedown.m b/CAE/caedown.m new file mode 100644 index 0000000..8e6736d --- /dev/null +++ b/CAE/caedown.m @@ -0,0 +1,13 @@ +function cae = caedown(cae) + pa = cae.a; + pok = cae.ok; + + for i = 1 : numel(cae.o) + z = 0; + for j = 1 : numel(cae.a) + z = z + convn(pa{j}, pok{i}{j}, 'valid'); + end + cae.o{i} = sigm(z + cae.c{i}); + + end +end diff --git a/SPAE/spaeexamples.m b/CAE/caeexamples.m similarity index 78% rename from SPAE/spaeexamples.m rename to CAE/caeexamples.m index 214ff07..65a188f 100644 --- a/SPAE/spaeexamples.m +++ b/CAE/caeexamples.m @@ -7,7 +7,7 @@ x{i}{1} = reshape(train_x(((i - 1) * N + 1) : (i) * N, :), N, 28, 28) * 255; end %% ex 1 -spae = { +scae = { struct('outputmaps', 10, 'inputkernel', [1 5 5], 'outputkernel', [1 5 5], 'scale', [1 2 2], 'sigma', 0.1, 'momentum', 0.9, 'noise', 0) }; @@ -16,17 +16,17 @@ opts.alpha = 0.01; opts.ddinterval = 10; opts.ddhist = 0.5; -spae = spaesetup(spae, x, opts); -spae = spaetrain(spae, x, opts); -pae = spae{1}; +scae = scaesetup(scae, x, opts); +scae = scaetrain(scae, x, opts); +cae = scae{1}; %Visualize the average reconstruction error -plot(pae.rL); +plot(cae.rL); %Visualize the output kernels ff=[]; -for i=1:numel(pae.ok{1}); - mm = pae.ok{1}{i}(1,:,:); +for i=1:numel(cae.ok{1}); + mm = cae.ok{1}{i}(1,:,:); ff(i,:) = mm(:); end; figure;visualize(ff',1) diff --git a/CAE/caenumgradcheck.m b/CAE/caenumgradcheck.m new file mode 100644 index 0000000..125d926 --- /dev/null +++ b/CAE/caenumgradcheck.m @@ -0,0 +1,107 @@ +function cae = caenumgradcheck(cae, x, y) + epsilon = 1e-4; + er = 1e-6; + disp('performing numerical gradient checking...') + for i = 1 : numel(cae.o) + p_cae = cae; p_cae.c{i} = p_cae.c{i} + epsilon; + m_cae = cae; m_cae.c{i} = m_cae.c{i} - epsilon; + + [m_cae, p_cae] = caerun(m_cae, p_cae, x, y); + d = (p_cae.L - m_cae.L) / (2 * epsilon); + + e = abs(d - cae.dc{i}); + if e > er + disp('OUTPUT BIAS numerical gradient checking failed'); + disp(e); + disp(d / cae.dc{i}); + keyboard + end + end + + for a = 1 : numel(cae.a) + + p_cae = cae; p_cae.b{a} = p_cae.b{a} + epsilon; + m_cae = cae; m_cae.b{a} = m_cae.b{a} - epsilon; + + [m_cae, p_cae] = caerun(m_cae, p_cae, x, y); + d = (p_cae.L - m_cae.L) / (2 * epsilon); +% cae.dok{i}{a}(u) = d; + e = abs(d - cae.db{a}); + if e > er + disp('BIAS numerical gradient checking failed'); + disp(e); + disp(d / cae.db{a}); + keyboard + end + + for i = 1 : numel(cae.o) + for u = 1 : numel(cae.ok{i}{a}) + p_cae = cae; p_cae.ok{i}{a}(u) = p_cae.ok{i}{a}(u) + epsilon; + m_cae = cae; m_cae.ok{i}{a}(u) = m_cae.ok{i}{a}(u) - epsilon; + + [m_cae, p_cae] = caerun(m_cae, p_cae, x, y); + d = (p_cae.L - m_cae.L) / (2 * epsilon); +% cae.dok{i}{a}(u) = d; + e = abs(d - cae.dok{i}{a}(u)); + if e > er + disp('OUTPUT KERNEL numerical gradient checking failed'); + disp(e); + disp(d / cae.dok{i}{a}(u)); +% keyboard + end + end + end + + for i = 1 : numel(cae.i) + for u = 1 : numel(cae.ik{i}{a}) + p_cae = cae; + m_cae = cae; + p_cae.ik{i}{a}(u) = p_cae.ik{i}{a}(u) + epsilon; + m_cae.ik{i}{a}(u) = m_cae.ik{i}{a}(u) - epsilon; + [m_cae, p_cae] = caerun(m_cae, p_cae, x, y); + d = (p_cae.L - m_cae.L) / (2 * epsilon); +% cae.dik{i}{a}(u) = d; + e = abs(d - cae.dik{i}{a}(u)); + if e > er + disp('INPUT KERNEL numerical gradient checking failed'); + disp(e); + disp(d / cae.dik{i}{a}(u)); + end + end + end + end + + disp('done') + +end + +function [m_cae, p_cae] = caerun(m_cae, p_cae, x, y) + m_cae = caeup(m_cae, x); m_cae = caedown(m_cae); m_cae = caebp(m_cae, y); + p_cae = caeup(p_cae, x); p_cae = caedown(p_cae); p_cae = caebp(p_cae, y); +end + +%function checknumgrad(cae,what,x,y) +% epsilon = 1e-4; +% er = 1e-9; +% +% for i = 1 : numel(eval(what)) +% if iscell(eval(['cae.' what])) +% checknumgrad(cae,[what '{' num2str(i) '}'], x, y) +% else +% p_cae = cae; +% m_cae = cae; +% eval(['p_cae.' what '(' num2str(i) ')']) = eval([what '(' num2str(i) ')']) + epsilon; +% eval(['m_cae.' what '(' num2str(i) ')']) = eval([what '(' num2str(i) ')']) - epsilon; +% +% m_cae = caeff(m_cae, x); m_cae = caedown(m_cae); m_cae = caebp(m_cae, y); +% p_cae = caeff(p_cae, x); p_cae = caedown(p_cae); p_cae = caebp(p_cae, y); +% +% d = (p_cae.L - m_cae.L) / (2 * epsilon); +% e = abs(d - eval(['cae.d' what '(' num2str(i) ')'])); +% if e > er +% error('numerical gradient checking failed'); +% end +% end +% end +% +% end diff --git a/CAE/caesdlm.m b/CAE/caesdlm.m new file mode 100644 index 0000000..2c7c22d --- /dev/null +++ b/CAE/caesdlm.m @@ -0,0 +1,26 @@ +function cae = caesdlm(cae, opts, m) + %stochastic diagonal levenberg-marquardt + + %first round + if isfield(cae,'ddok') == 0 + cae = caebbp(cae); + end + + %recalculate double grads every opts.ddinterval + if mod(m, opts.ddinterval) == 0 + cae_n = caebbp(cae); + + for ii = 1 : numel(cae.o) + cae.ddc{ii} = opts.ddhist * cae.ddc{ii} + (1 - opts.ddhist) * cae_n.ddc{ii}; + end + + for jj = 1 : numel(cae.a) + cae.ddb{jj} = opts.ddhist * cae.ddb{jj} + (1 - opts.ddhist) * cae_n.ddb{jj}; + for ii = 1 : numel(cae.o) + cae.ddok{ii}{jj} = opts.ddhist * cae.ddok{ii}{jj} + (1 - opts.ddhist) * cae_n.ddok{ii}{jj}; + cae.ddik{ii}{jj} = opts.ddhist * cae.ddik{ii}{jj} + (1 - opts.ddhist) * cae_n.ddik{ii}{jj}; + end + end + + end +end \ No newline at end of file diff --git a/SPAE/paetrain.m b/CAE/caetrain.m similarity index 57% rename from SPAE/paetrain.m rename to CAE/caetrain.m index 8223010..0751728 100644 --- a/SPAE/paetrain.m +++ b/CAE/caetrain.m @@ -1,6 +1,6 @@ -function pae = paetrain(pae, x, opts) - n = pae.inputkernel(1); - pae.rL = []; +function cae = caetrain(cae, x, opts) + n = cae.inputkernel(1); + cae.rL = []; for m = 1 : opts.rounds tic; disp([num2str(m) '/' num2str(opts.rounds) ' rounds']); @@ -14,21 +14,21 @@ x2{1} = double(x{i1}{1}(l + n : l + n + opts.batchsize - 1, :, :)) / 255; end % Add noise to input, for denoising stacked autoenoder - x1{1} = x1{1} .* (rand(size(x1{1})) > pae.noise); + x1{1} = x1{1} .* (rand(size(x1{1})) > cae.noise); - pae = paeup(pae, x1); - pae = paedown(pae); - pae = paebp(pae, x2); - pae = paesdlm(pae, opts, m); -% paenumgradcheck(pae,x1,x2); - pae = paeapplygrads(pae); + cae = caeup(cae, x1); + cae = caedown(cae); + cae = caebp(cae, x2); + cae = caesdlm(cae, opts, m); +% caenumgradcheck(cae,x1,x2); + cae = caeapplygrads(cae); if m == 1 - pae.rL(1) = pae.L; + cae.rL(1) = cae.L; end -% pae.rL(m + 1) = 0.99 * pae.rL(m) + 0.01 * pae.L; - pae.rL(m + 1) = pae.L; -% if pae.sv < 1e-10 +% cae.rL(m + 1) = 0.99 * cae.rL(m) + 0.01 * cae.L; + cae.rL(m + 1) = cae.L; +% if cae.sv < 1e-10 % disp('Converged'); % break; % end diff --git a/CAE/caeup.m b/CAE/caeup.m new file mode 100644 index 0000000..52c076e --- /dev/null +++ b/CAE/caeup.m @@ -0,0 +1,25 @@ +function cae = caeup(cae, x) + cae.i = x; + + %init temp vars for parrallel processing + pa = cell(size(cae.a)); + pi = cae.i; + pik = cae.ik; + pb = cae.b; + + for j = 1 : numel(cae.a) + z = 0; + for i = 1 : numel(pi) + z = z + convn(pi{i}, pik{i}{j}, 'full'); + end + pa{j} = sigm(z + pb{j}); + + % Max pool. + if ~isequal(cae.scale, [1 1 1]) + pa{j} = max3d(pa{j}, cae.M); + end + + end + cae.a = pa; + +end diff --git a/SPAE/max3d.m b/CAE/max3d.m similarity index 100% rename from SPAE/max3d.m rename to CAE/max3d.m diff --git a/CAE/scaesetup.m b/CAE/scaesetup.m new file mode 100644 index 0000000..5ede971 --- /dev/null +++ b/CAE/scaesetup.m @@ -0,0 +1,58 @@ +function scae = scaesetup(cae, x, opts) + x = x{1}; + for l = 1 : numel(cae) + cae = cae{l}; + ll= [opts.batchsize size(x{1}, 2) size(x{1}, 3)] + cae.inputkernel - 1; + X = zeros(ll); + cae.M = nbmap(X, cae.scale); + bounds = cae.outputmaps * prod(cae.inputkernel) + numel(x) * prod(cae.outputkernel); + for j = 1 : cae.outputmaps % activation maps + cae.a{j} = zeros(size(x{1}) + cae.inputkernel - 1); + for i = 1 : numel(x) % input map + cae.ik{i}{j} = (rand(cae.inputkernel) - 0.5) * 2 * sqrt(6 / bounds); + cae.ok{i}{j} = (rand(cae.outputkernel) - 0.5) * 2 * sqrt(6 / bounds); + cae.vik{i}{j} = zeros(size(cae.ik{i}{j})); + cae.vok{i}{j} = zeros(size(cae.ok{i}{j})); + end + cae.b{j} = 0; + cae.vb{j} = zeros(size(cae.b{j})); + end + + cae.alpha = opts.alpha; + + cae.i = cell(numel(x), 1); + cae.o = cae.i; + + for i = 1 : numel(cae.o) + cae.c{i} = 0; + cae.vc{i} = zeros(size(cae.c{i})); + end + + ss = cae.outputkernel; + + cae.edgemask = zeros([opts.batchsize size(x{1}, 2) size(x{1}, 3)]); + + cae.edgemask(ss(1) : end - ss(1) + 1, ... + ss(2) : end - ss(2) + 1, ... + ss(3) : end - ss(3) + 1) = 1; + + scae{l} = cae; + end + + function B = nbmap(X,n) + assert(numel(n)==3,'n should have 3 elements (x,y,z) scaling.'); + X = reshape(1:numel(X),size(X,1),size(X,2),size(X,3)); + B = zeros(size(X,1)/n(1),prod(n),size(X,2)*size(X,3)/prod(n(2:3))); + u=1; + p=1; + for m=1:size(X,1) + B(u,(p-1)*prod(n(2:3))+1:p*prod(n(2:3)),:) = im2col(squeeze(X(m,:,:)),n(2:3),'distinct'); + p=p+1; + if(mod(m,n(1))==0) + u=u+1; + p=1; + end + end + + end +end diff --git a/CAE/scaetrain.m b/CAE/scaetrain.m new file mode 100644 index 0000000..e249656 --- /dev/null +++ b/CAE/scaetrain.m @@ -0,0 +1,8 @@ +function scae = scaetrain(scae, x, opts) + %TODO: Transform x through scae{1} into new x. Only works for a single PAE. +% for i=1:numel(scae) +% scae{i} = paetrain(scae{i}, x, opts); +% end + scae{1} = caetrain(scae{1}, x, opts); + +end \ No newline at end of file diff --git a/SPAE/nbmap.m b/SPAE/nbmap.m deleted file mode 100644 index c2afb46..0000000 --- a/SPAE/nbmap.m +++ /dev/null @@ -1,16 +0,0 @@ -function B = nbmap(X,n) - assert(numel(n)==3,'n should have 3 elements (x,y,z) scaling.'); - X = reshape(1:numel(X),size(X,1),size(X,2),size(X,3)); - B = zeros(size(X,1)/n(1),prod(n),size(X,2)*size(X,3)/prod(n(2:3))); - u=1; - p=1; - for i=1:size(X,1) - B(u,(p-1)*prod(n(2:3))+1:p*prod(n(2:3)),:) = im2col(squeeze(X(i,:,:)),n(2:3),'distinct'); - p=p+1; - if(mod(i,n(1))==0) - u=u+1; - p=1; - end - end - -end \ No newline at end of file diff --git a/SPAE/paeapplygrads.m b/SPAE/paeapplygrads.m deleted file mode 100644 index 3e4008e..0000000 --- a/SPAE/paeapplygrads.m +++ /dev/null @@ -1,29 +0,0 @@ -function pae = paeapplygrads(pae) - pae.sv = 0; - for j = 1 : numel(pae.a) - for i = 1 : numel(pae.i) -% pae.vik{i}{j} = pae.momentum * pae.vik{i}{j} + pae.alpha ./ (pae.sigma + pae.ddik{i}{j}) .* pae.dik{i}{j}; -% pae.vok{i}{j} = pae.momentum * pae.vok{i}{j} + pae.alpha ./ (pae.sigma + pae.ddok{i}{j}) .* pae.dok{i}{j}; - pae.vik{i}{j} = pae.alpha * pae.dik{i}{j}; - pae.vok{i}{j} = pae.alpha * pae.dok{i}{j}; - pae.sv = pae.sv + sum(pae.vik{i}{j}(:) .^ 2); - pae.sv = pae.sv + sum(pae.vok{i}{j}(:) .^ 2); - - pae.ik{i}{j} = pae.ik{i}{j} - pae.vik{i}{j}; - pae.ok{i}{j} = pae.ok{i}{j} - pae.vok{i}{j}; - end -% pae.vb{j} = pae.momentum * pae.vb{j} + pae.alpha / (pae.sigma + pae.ddb{j}) * pae.db{j}; - pae.vb{j} = pae.alpha * pae.db{j}; - pae.sv = pae.sv + sum(pae.vb{j} .^ 2); - - pae.b{j} = pae.b{j} - pae.vb{j}; - end - - for i = 1 : numel(pae.o) -% pae.vc{i} = pae.momentum * pae.vc{i} + pae.alpha / (pae.sigma + pae.ddc{i}) * pae.dc{i}; - pae.vc{i} = pae.alpha * pae.dc{i}; - pae.sv = pae.sv + sum(pae.vc{i} .^ 2); - - pae.c{i} = pae.c{i} - pae.vc{i}; - end -end diff --git a/SPAE/paebbp.m b/SPAE/paebbp.m deleted file mode 100644 index c7d8860..0000000 --- a/SPAE/paebbp.m +++ /dev/null @@ -1,29 +0,0 @@ -function pae = paebbp(pae) - - %% backprop deltas - for i = 1 : numel(pae.o) - % output delta delta - pae.odd{i} = (pae.o{i} .* (1 - pae.o{i}) .* pae.edgemask) .^ 2; - % delta delta c - pae.ddc{i} = sum(pae.odd{i}(:)) / size(pae.odd{i}, 1); - end - - for j = 1 : numel(pae.a) % calc activation delta deltas - z = 0; - for i = 1 : numel(pae.o) - z = z + convn(pae.odd{i}, flipall(pae.ok{i}{j} .^ 2), 'full'); - end - pae.add{j} = (pae.a{j} .* (1 - pae.a{j})) .^ 2 .* z; - end - - %% calc params delta deltas - ns = size(pae.odd{1}, 1); - for j = 1 : numel(pae.a) - pae.ddb{j} = sum(pae.add{j}(:)) / ns; - for i = 1 : numel(pae.o) - pae.ddok{i}{j} = convn(flipall(pae.a{j} .^ 2), pae.odd{i}, 'valid') / ns; - pae.ddik{i}{j} = convn(pae.add{j}, flipall(pae.i{i} .^ 2), 'valid') / ns; - end - end - -end diff --git a/SPAE/paebp.m b/SPAE/paebp.m deleted file mode 100644 index 421f6be..0000000 --- a/SPAE/paebp.m +++ /dev/null @@ -1,34 +0,0 @@ -function pae = paebp(pae, y) - - %% backprop deltas - pae.L = 0; - for i = 1 : numel(pae.o) - % error - pae.e{i} = (pae.o{i} - y{i}) .* pae.edgemask; - % loss function - pae.L = pae.L + 1/2 * sum(pae.e{i}(:) .^2 ) / size(pae.e{i}, 1); - % output delta - pae.od{i} = pae.e{i} .* (pae.o{i} .* (1 - pae.o{i})); - - pae.dc{i} = sum(pae.od{i}(:)) / size(pae.e{i}, 1); - end - - for j = 1 : numel(pae.a) % calc activation deltas - z = 0; - for i = 1 : numel(pae.o) - z = z + convn(pae.od{i}, flipall(pae.ok{i}{j}), 'full'); - end - pae.ad{j} = pae.a{j} .* (1 - pae.a{j}) .* z; - end - - %% calc gradients - ns = size(pae.e{1}, 1); - for j = 1 : numel(pae.a) - pae.db{j} = sum(pae.ad{j}(:)) / ns; - for i = 1 : numel(pae.o) - pae.dok{i}{j} = convn(flipall(pae.a{j}), pae.od{i}, 'valid') / ns; - pae.dik{i}{j} = convn(pae.ad{j}, flipall(pae.i{i}), 'valid') / ns; - end - end - -end diff --git a/SPAE/paedown.m b/SPAE/paedown.m deleted file mode 100644 index 82af60d..0000000 --- a/SPAE/paedown.m +++ /dev/null @@ -1,13 +0,0 @@ -function pae = paedown(pae) - pa = pae.a; - pok = pae.ok; - - for i = 1 : numel(pae.o) - z = 0; - for j = 1 : numel(pae.a) - z = z + convn(pa{j}, pok{i}{j}, 'valid'); - end - pae.o{i} = sigm(z + pae.c{i}); - - end -end diff --git a/SPAE/paenumgradcheck.m b/SPAE/paenumgradcheck.m deleted file mode 100644 index 1f36fba..0000000 --- a/SPAE/paenumgradcheck.m +++ /dev/null @@ -1,107 +0,0 @@ -function pae = paenumgradcheck(pae, x, y) - epsilon = 1e-4; - er = 1e-6; - disp('performing numerical gradient checking...') - for i = 1 : numel(pae.o) - p_pae = pae; p_pae.c{i} = p_pae.c{i} + epsilon; - m_pae = pae; m_pae.c{i} = m_pae.c{i} - epsilon; - - [m_pae, p_pae] = paerun(m_pae, p_pae, x, y); - d = (p_pae.L - m_pae.L) / (2 * epsilon); - - e = abs(d - pae.dc{i}); - if e > er - disp('OUTPUT BIAS numerical gradient checking failed'); - disp(e); - disp(d / pae.dc{i}); - keyboard - end - end - - for a = 1 : numel(pae.a) - - p_pae = pae; p_pae.b{a} = p_pae.b{a} + epsilon; - m_pae = pae; m_pae.b{a} = m_pae.b{a} - epsilon; - - [m_pae, p_pae] = paerun(m_pae, p_pae, x, y); - d = (p_pae.L - m_pae.L) / (2 * epsilon); -% pae.dok{i}{a}(u) = d; - e = abs(d - pae.db{a}); - if e > er - disp('BIAS numerical gradient checking failed'); - disp(e); - disp(d / pae.db{a}); - keyboard - end - - for i = 1 : numel(pae.o) - for u = 1 : numel(pae.ok{i}{a}) - p_pae = pae; p_pae.ok{i}{a}(u) = p_pae.ok{i}{a}(u) + epsilon; - m_pae = pae; m_pae.ok{i}{a}(u) = m_pae.ok{i}{a}(u) - epsilon; - - [m_pae, p_pae] = paerun(m_pae, p_pae, x, y); - d = (p_pae.L - m_pae.L) / (2 * epsilon); -% pae.dok{i}{a}(u) = d; - e = abs(d - pae.dok{i}{a}(u)); - if e > er - disp('OUTPUT KERNEL numerical gradient checking failed'); - disp(e); - disp(d / pae.dok{i}{a}(u)); -% keyboard - end - end - end - - for i = 1 : numel(pae.i) - for u = 1 : numel(pae.ik{i}{a}) - p_pae = pae; - m_pae = pae; - p_pae.ik{i}{a}(u) = p_pae.ik{i}{a}(u) + epsilon; - m_pae.ik{i}{a}(u) = m_pae.ik{i}{a}(u) - epsilon; - [m_pae, p_pae] = paerun(m_pae, p_pae, x, y); - d = (p_pae.L - m_pae.L) / (2 * epsilon); -% pae.dik{i}{a}(u) = d; - e = abs(d - pae.dik{i}{a}(u)); - if e > er - disp('INPUT KERNEL numerical gradient checking failed'); - disp(e); - disp(d / pae.dik{i}{a}(u)); - end - end - end - end - - disp('done') - -end - -function [m_pae, p_pae] = paerun(m_pae, p_pae, x, y) - m_pae = paeup(m_pae, x); m_pae = paedown(m_pae); m_pae = paebp(m_pae, y); - p_pae = paeup(p_pae, x); p_pae = paedown(p_pae); p_pae = paebp(p_pae, y); -end - -%function checknumgrad(pae,what,x,y) -% epsilon = 1e-4; -% er = 1e-9; -% -% for i = 1 : numel(eval(what)) -% if iscell(eval(['pae.' what])) -% checknumgrad(pae,[what '{' num2str(i) '}'], x, y) -% else -% p_pae = pae; -% m_pae = pae; -% eval(['p_pae.' what '(' num2str(i) ')']) = eval([what '(' num2str(i) ')']) + epsilon; -% eval(['m_pae.' what '(' num2str(i) ')']) = eval([what '(' num2str(i) ')']) - epsilon; -% -% m_pae = paeff(m_pae, x); m_pae = paedown(m_pae); m_pae = paebp(m_pae, y); -% p_pae = paeff(p_pae, x); p_pae = paedown(p_pae); p_pae = paebp(p_pae, y); -% -% d = (p_pae.L - m_pae.L) / (2 * epsilon); -% e = abs(d - eval(['pae.d' what '(' num2str(i) ')'])); -% if e > er -% error('numerical gradient checking failed'); -% end -% end -% end -% -% end diff --git a/SPAE/paesdlm.m b/SPAE/paesdlm.m deleted file mode 100644 index e7551e3..0000000 --- a/SPAE/paesdlm.m +++ /dev/null @@ -1,26 +0,0 @@ -function pae = paesdlm(pae, opts, m) - %stochastic diagonal levenberg-marquardt - - %first round - if isfield(pae,'ddok') == 0 - pae = paebbp(pae); - end - - %recalculate double grads every opts.ddinterval - if mod(m, opts.ddinterval) == 0 - pae_n = paebbp(pae); - - for ii = 1 : numel(pae.o) - pae.ddc{ii} = opts.ddhist * pae.ddc{ii} + (1 - opts.ddhist) * pae_n.ddc{ii}; - end - - for jj = 1 : numel(pae.a) - pae.ddb{jj} = opts.ddhist * pae.ddb{jj} + (1 - opts.ddhist) * pae_n.ddb{jj}; - for ii = 1 : numel(pae.o) - pae.ddok{ii}{jj} = opts.ddhist * pae.ddok{ii}{jj} + (1 - opts.ddhist) * pae_n.ddok{ii}{jj}; - pae.ddik{ii}{jj} = opts.ddhist * pae.ddik{ii}{jj} + (1 - opts.ddhist) * pae_n.ddik{ii}{jj}; - end - end - - end -end \ No newline at end of file diff --git a/SPAE/paeup.m b/SPAE/paeup.m deleted file mode 100644 index af82f87..0000000 --- a/SPAE/paeup.m +++ /dev/null @@ -1,25 +0,0 @@ -function pae = paeup(pae, x) - pae.i = x; - - %init temp vars for parrallel processing - pa = cell(size(pae.a)); - pi = pae.i; - pik = pae.ik; - pb = pae.b; - - for j = 1 : numel(pae.a) - z = 0; - for i = 1 : numel(pi) - z = z + convn(pi{i}, pik{i}{j}, 'full'); - end - pa{j} = sigm(z + pb{j}); - - % Max pool. - if ~isequal(pae.scale, [1 1 1]) - pa{j} = max3d(pa{j}, pae.M); - end - - end - pae.a = pa; - -end diff --git a/SPAE/spaesetup.m b/SPAE/spaesetup.m deleted file mode 100644 index 7e82b5a..0000000 --- a/SPAE/spaesetup.m +++ /dev/null @@ -1,41 +0,0 @@ -function spae = spaesetup(spae, x, opts) - x = x{1}; - for l = 1 : numel(spae) - pae = spae{l}; - ll= [opts.batchsize size(x{1}, 2) size(x{1}, 3)] + pae.inputkernel - 1; - X = zeros(ll); - pae.M = nbmap(X, pae.scale); - bounds = pae.outputmaps * prod(pae.inputkernel) + numel(x) * prod(pae.outputkernel); - for j = 1 : pae.outputmaps % activation maps - pae.a{j} = zeros(size(x{1}) + pae.inputkernel - 1); - for i = 1 : numel(x) % input map - pae.ik{i}{j} = (rand(pae.inputkernel) - 0.5) * 2 * sqrt(6 / bounds); - pae.ok{i}{j} = (rand(pae.outputkernel) - 0.5) * 2 * sqrt(6 / bounds); - pae.vik{i}{j} = zeros(size(pae.ik{i}{j})); - pae.vok{i}{j} = zeros(size(pae.ok{i}{j})); - end - pae.b{j} = 0; - pae.vb{j} = zeros(size(pae.b{j})); - end - - pae.alpha = opts.alpha; - - pae.i = cell(numel(x), 1); - pae.o = pae.i; - - for i = 1 : numel(pae.o) - pae.c{i} = 0; - pae.vc{i} = zeros(size(pae.c{i})); - end - - ss = pae.outputkernel; - - pae.edgemask = zeros([opts.batchsize size(x{1}, 2) size(x{1}, 3)]); - - pae.edgemask(ss(1) : end - ss(1) + 1, ... - ss(2) : end - ss(2) + 1, ... - ss(3) : end - ss(3) + 1) = 1; - - spae{l} = pae; - end -end diff --git a/SPAE/spaetrain.m b/SPAE/spaetrain.m deleted file mode 100644 index 6418cae..0000000 --- a/SPAE/spaetrain.m +++ /dev/null @@ -1,8 +0,0 @@ -function spae = spaetrain(spae, x, opts) - %TODO: Transform x through spae{1} into new x. Only works for a single PAE. -% for i=1:numel(spae) -% spae{i} = paetrain(spae{i}, x, opts); -% end - spae{1} = paetrain(spae{1}, x, opts); - -end \ No newline at end of file