forms/run_forms_new_images.m at master · jmmanley/forms · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
%% EXAMPLE SCRIPT FOR FITTING A 3D MORPHABLE MODEL TO A SET OF IMAGES
%
% Jason Manley, updated Mar 2018
% Question? jmanley@rockefeller.edu

addpath(genpath('/path/to/repo/forms/'))

path = '/path/to/your/images/';
file_type = '.your_file_type';

files = dir(fullfile(path,['*' file_type]));
N = length(files);

project = struct();


%% load your own mesh variables, or use one of those provided by Cashman & Fitzgibbon

% example: use their dolphins mesh
dolphins = read_project('projects/dolphins.fpj');

project.vertices    = dolphins.vertices;
project.mesh        = dolphins.mesh;
project.faces       = dolphins.faces;
project.cand_ixs    = dolphins.cand_ixs;
project.cand_uvs    = dolphins.cand_uvs;
project.cand_limits = dolphins.cand_limits;
project.cand_dists  = dolphins.cand_dists;
project.cand_derivs = dolphins.cand_derivs;


%% load images

project.images = struct();

for i=1:length(files)
    project.images(i).image = imread(files(i).name);
end


%% find silhouettes
%  here you should use a method for finding the silhouette of your object
%  of interest.

% simple example: convert images to grayscale, threshold images, find
% silhouette around largest region

threshold = 20; % this will need to be tweaked for your images...

figure;
for i=1:N
    grayscale = rgb2gray(project.images(i).image);
    props = regionprops(grayscale>threshold,'PixelIdxList','Area');
    [~,sortIdx] = sort(props.Area,'descend');

    % find largest region
    mask = zeros(size(grayscale));
    mask(props(sortIdx(1)).PixelIdxList) = 1;

    % find silhouette
    [i,j] = ind2sub(size(mask),props(sortIdx(1)).PixelIdxList(1));
    project.images(i).silhouette = bwtraceboundary(mask,[i j], 'E');

    % plot result
    clf;
    imagesc(project.images(i).image); title(['Image ' num2str(i)])
    plot(project.images(i).silhouette(:,2),images(i).silhouette(:,1),'r','LineWidth',2);
    pause;
end


%% set initial conditions for all images

for i=1:N
    dolphins = set_initial_conditions(project,i);
end

% if curious, plot all dolphins with silhouettes and constraint points
f = figure;
for i=1:N
    clf;
    imshow(project.images(i).image,'Border','tight'); title(['Image ' num2str(i)])
    hold on; plot(project.images(i).silhouette(:,1),project.images(i).silhouette(:,2),'r','linewidth',3)
    scatter(project.images(i).constraints2d(:,1),project.images(i).constraints2d(:,2),'w','filled')
end

% if curious, plot initial conditions
figure;
for i=1:N
    clf;
    plot_modelfit(project,i); title(['Image ' num2str(i)])
    pause;
end


%% fit the model!

parameters = makeModelParameters; % likely should test various parameter values!

model = forms(project, parameters);


%% visualize model fits

% model fits by image
figure;
for i=1:N
     clf;
    subplot(1,3,1); imagesc(project.images(i).image);
    hold on; scatter(project.images(i).silhouette(:,1),project.images(i).silhouette(:,2),10,'r','filled')
    title('Original Frame','fontsize',14); axis off;

    a=subplot(1,3,2); plot_modelfit(project,i,[],true,false); axis off;
    title('Initial Conditions','fontsize',14);

    b=subplot(1,3,3); plot_modelfit(project,i,dolphin_model,true,false); axis off;
    title('Model Fit','fontsize',14)
    linkaxes([a,b],'xy');
    Link = linkprop([a, b], ...
       {'CameraUpVector', 'CameraPosition', 'CameraTarget'});
    setappdata(gcf, 'StoreTheLink', Link);
    pause;
end

% basis shapes
figure;
for i=1:parameters.M
    plot_basisshapenorm(dolphins,dolphin_model,i); title(['Basis Shape ' num2str(i)])
    pause;
end