History

initial release

Yusong Gao committed 2 years ago

Commit d0d49a9No parent (first commit)21 files · +8268 −0Browse files

Showing the changes to test_tensor.cpp (1 of 21 files). Show all

Unified · Split

test_tensor.cpp+610−0
@@ -0,0 +1,610 @@
1#include <iostream>
2
3#include "boost/ut.hpp"
4#include "tinytorch.hpp"
5
6using namespace tinytorch; // NOLINT
7using namespace boost; // NOLINT
8
9namespace std {
10
11template <typename T>
12ostream &operator<<(ostream &os, const vector<T> &vec) {
13 os << "[";
14 for (size_t i = 0; i < vec.size(); i++) {
15 os << vec[i];
16 if (i < vec.size() - 1) {
17 os << ", ";
18 }
19 }
20 os << "]";
21 return os;
22}
23
24}; // namespace std
25
26#define TO_VEC(a) std::vector<float>(a, a + sizeof(a) / sizeof(float))
27
28const float eps = 1e-5; // NOLINT
29
30bool CheckVectorEqual(const std::vector<float> &a, const std::vector<float> &b) {
31 if (a.size() != b.size()) {
32 return false;
33 }
34 for (size_t i = 0; i < a.size(); i++) {
35 if (std::isnan(a[i]) || std::isnan(b[i]) || fabs(a[i] - b[i]) > eps) {
36 return false;
37 }
38 }
39 return true;
40}
41
42void TestTensor(TensorContext &ctx) {
43 ut::test("TestTensorAdd") = [&ctx] {
44 std::vector<int> a_dims = {2, 3, 2};
45 float a_data[] = {-0.156085f, 0.119778f, 0.761198f, 0.347593f, -0.474833f, -0.314762f,
46 -0.510900f, -0.273722f, -0.890527f, 0.362633f, -0.958626f, 0.358971f};
47 std::vector<int> b_dims = {2, 3, 2};
48 float b_data[] = {0.642503f, 0.716670f, -0.840944f, -0.951772f, -0.858585f, 0.895501f,
49 -0.205840f, -0.693812f, -0.889058f, 0.434394f, 0.890442f, 0.936250f};
50 std::vector<int> result_dims = {2, 3, 2};
51 float result_data[] = {0.486419f, 0.836448f, -0.079746f, -0.604179f, -1.333419f, 0.580739f,
52 -0.716740f, -0.967534f, -1.779585f, 0.797027f, -0.068184f, 1.295222f};
53
54 auto &a = *ctx.NewTensor(a_dims)->Fill(a_data);
55 auto &b = *ctx.NewTensor(b_dims)->Fill(b_data);
56 auto &c = a + b;
57 c.Forward();
58 ut::expect(CheckVectorEqual(c.Flatten(), TO_VEC(result_data)))
59 << c.Flatten() << " != " << TO_VEC(result_data) << "\n";
60 };
61
62 ut::test("TestTensorMul") = [&ctx] {
63 std::vector<int> a_dims = {2, 3, 2};
64 float a_data[] = {0.321711f, 0.549407f, -0.011015f, -0.610029f, -0.721674f, -0.857888f,
65 -0.992038f, 0.380269f, 0.134258f, -0.639243f, -0.338716f, 0.289208f};
66 float a_grad_data[] = {0.110795f, -0.960914f, 0.314134f, -0.145329f, -0.071354f, -0.052769f,
67 0.418165f, -0.533604f, -0.073091f, 0.607927f, 0.679904f, 0.377298f};
68 std::vector<int> b_dims = {2, 3, 2};
69 float b_data[] = {0.110795f, -0.960914f, 0.314134f, -0.145329f, -0.071354f, -0.052769f,
70 0.418165f, -0.533604f, -0.073091f, 0.607927f, 0.679904f, 0.377298f};
71 float b_grad_data[] = {0.321711f, 0.549407f, -0.011015f, -0.610029f, -0.721674f, -0.857888f,
72 -0.992038f, 0.380269f, 0.134258f, -0.639243f, -0.338716f, 0.289208f};
73 std::vector<int> result_dims = {2, 3, 2};
74 float result_data[] = {0.035644f, -0.527933f, -0.003460f, 0.088655f, 0.051494f, 0.045270f,
75 -0.414835f, -0.202913f, -0.009813f, -0.388613f, -0.230294f, 0.109118f};
76 auto &a = *ctx.NewTensor(a_dims)->Fill(a_data);
77 auto &b = *ctx.NewTensor(b_dims)->Fill(b_data);
78 auto b_grad = b.RandomGrad()->grad();
79 auto &b_grad_truth = *ctx.NewTensor(b_dims)->Fill(b_grad_data) + *b_grad;
80 b_grad_truth.Forward();
81 auto &c = a * b;
82 c.Forward();
83 c.Backward();
84
85 ut::expect(CheckVectorEqual(c.Flatten(), TO_VEC(result_data)))
86 << c.Flatten() << " != " << TO_VEC(result_data) << "\n";
87 ut::expect(CheckVectorEqual(a.grad()->Flatten(), TO_VEC(a_grad_data)))
88 << "a grad: " << a.grad()->Flatten() << " != " << TO_VEC(a_grad_data) << "\n";
89 ut::expect(CheckVectorEqual(b.grad()->Flatten(), b_grad_truth.Flatten()))
90 << "b grad: " << b.grad()->Flatten() << " != " << b_grad_truth.Flatten() << "\n";
91 };
92
93 ut::test("TestTensorMulMat") = [&ctx] {
94 std::vector<int> a_dims = {2, 3, 2};
95 float a_data[] = {0.038856f, -0.205084f, -0.576147f, -0.253080f, -0.973295f, 0.236236f,
96 0.515430f, 0.641762f, -0.492475f, -0.325701f, -0.156254f, 0.507203f};
97 std::vector<int> b_dims = {2, 4, 2};
98 float b_data[] = {-0.926039f, 0.038612f, 0.818189f, 0.535480f, 0.217580f, 0.744733f, 0.888028f, -0.513070f,
99 -0.341174f, -0.588775f, -0.814142f, 0.241264f, -0.224577f, -0.357855f, 0.081625f, -0.727418f};
100 std::vector<int> result_dims = {2, 3, 4};
101 float result_data[] = {-0.043901f, -0.078027f, -0.144279f, 0.139727f, 0.523762f, -0.606916f,
102 -0.313835f, -0.381786f, 0.910431f, -0.669840f, -0.035837f, -0.985519f,
103 -0.553705f, -0.264799f, -0.345412f, -0.424757f, 0.359784f, 0.322365f,
104 0.227153f, 0.196722f, -0.245319f, 0.249583f, -0.146414f, -0.381703f};
105 auto &a = *ctx.NewTensor(a_dims)->Fill(a_data);
106 auto &b = *ctx.NewTensor(b_dims)->Fill(b_data);
107 auto &c = a.MatMul(b);
108 c.Forward();
109 ut::expect(CheckVectorEqual(c.Flatten(), TO_VEC(result_data)))
110 << c.Flatten() << " != " << TO_VEC(result_data) << "\n";
111 };
112
113 ut::test("TestTensorLinear") = [&ctx] {
114 std::vector<int> a_dims = {2, 3, 2};
115 float a_data[] = {-0.124705f, 0.461944f, -0.035526f, 0.603655f, -0.565689f, 0.580007f,
116 0.739768f, 0.555825f, 0.353833f, -0.393867f, -0.692182f, 0.200951f};
117 float a_grad_data[] = {-0.334757f, -0.358625f, -0.334757f, -0.358625f, -0.334757f, -0.358625f,
118 0.461304f, -0.141092f, 0.461304f, -0.141092f, 0.461304f, -0.141092f};
119 std::vector<int> b_dims = {2, 4, 2};
120 float b_data[] = {-0.337031f, -0.849364f, 0.880950f, 0.673869f, -0.806584f, 0.693106f,
121 -0.072092f, -0.876236f, -0.607369f, 0.573827f, -0.175670f, -0.252041f,
122 0.433276f, -0.186126f, 0.811068f, -0.276751f};
123 float b_grad_data[] = {-0.725921f, 1.645606f, -0.725921f, 1.645606f, -0.725921f, 1.645606f,
124 -0.725921f, 1.645606f, 0.401419f, 0.362910f, 0.401419f, 0.362910f,
125 0.401419f, 0.362910f, 0.401419f, 0.362910f};
126 std::vector<int> c_dims = {2, 3, 4};
127 float c_data[] = {0.405725f, -0.488369f, 0.916681f, -0.544182f, 0.213521f, -0.491035f, 0.491596f, -0.358687f,
128 0.530066f, -0.208435f, -0.675905f, 0.453980f, -0.926804f, -0.315975f, 0.534358f, -0.848598f,
129 -0.431841f, -0.352661f, 0.723182f, 0.930128f, -0.483508f, -0.560935f, 0.256884f, 0.824113f};
130 float c_grad_data[] = {1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f,
131 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f,
132 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f};
133 std::vector<int> result_dims = {2, 3, 4};
134 float result_data[] = {0.055396f, -0.286938f, 1.337443f, -0.939964f, -0.287228f, -0.115548f,
135 0.938648f, -0.885070f, 0.228083f, -0.315930f, 0.182377f, -0.013461f,
136 -1.057168f, -0.586021f, 0.751428f, -0.402421f, -0.872759f, -0.315548f,
137 0.949798f, 1.326114f, 0.052213f, -0.489987f, -0.080424f, 0.207093f};
138 float result_grad_data[] = {1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f,
139 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f,
140 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f,
141 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f};
142
143 auto &a = *ctx.NewTensor(a_dims)->Fill(a_data);
144 auto &b = *ctx.NewTensor(b_dims)->Fill(b_data);
145 auto &c = *ctx.NewTensor(c_dims)->Fill(c_data);
146 auto &d = a.MatMul(b) + c;
147 d.Forward();
148 d.ZeroGrad();
149 d.Backward();
150
151 // std::cout << "Tensor a: " << std::endl;
152 // a.PrintTensor();
153 // std::cout << "Tensor b: " << std::endl;
154 // b.PrintTensor();
155 // std::cout << "Tensor c: " << std::endl;
156 // c.PrintTensor();
157 // std::cout << "Tensor d: " << std::endl;
158 // d.PrintTensor();
159
160 ut::expect(CheckVectorEqual(d.Flatten(), TO_VEC(result_data)))
161 << d.Flatten() << " != " << TO_VEC(result_data) << "\n";
162 ut::expect(CheckVectorEqual(d.grad()->Flatten(), TO_VEC(result_grad_data)))
163 << "result grad: " << d.grad()->Flatten() << " != " << TO_VEC(result_grad_data) << "\n";
164 ut::expect(CheckVectorEqual(a.grad()->Flatten(), TO_VEC(a_grad_data)))
165 << "a grad: " << a.grad()->Flatten() << " != " << TO_VEC(a_grad_data) << "\n";
166 ut::expect(CheckVectorEqual(b.grad()->Flatten(), TO_VEC(b_grad_data)))
167 << "b grad: " << b.grad()->Flatten() << " != " << TO_VEC(b_grad_data) << "\n";
168 ut::expect(CheckVectorEqual(c.grad()->Flatten(), TO_VEC(c_grad_data)))
169 << "c grad: " << c.grad()->Flatten() << " != " << TO_VEC(c_grad_data) << "\n";
170 };
171
172 ut::test("TestTensorLookup") = [&ctx] {
173 int32_t a_data[] = {2, 1, 2, 0, 3, 4};
174 auto a = ctx.NewTensor({2, 3}, kI32)->Fill(a_data);
175 float b_data[] = {0.642503f, 0.716670f, -0.840944f, -0.951772f, -0.858585f, 0.895501f,
176 -0.205840f, -0.693812f, -0.889058f, 0.434394f, 0.890442f, 0.936250f};
177 float b_grad_data[] = {1.0f, 1.0f, 1.0f, 1.0f, 2.0f, 2.0f, 1.0f, 1.0f, 1.0f, 1.0f, 0.0f, 0.0f};
178 auto b = ctx.NewTensor({6, 2})->Fill(b_data);
179
180 float result_data[] = {-0.858585f, 0.895501f, -0.840944f, -0.951772f, -0.858585f, 0.895501f,
181 0.642503f, 0.716670f, -0.205840f, -0.693812f, -0.889058f, 0.434394f};
182
183 auto &c = (*b)[*a];
184
185 c.Forward();
186 c.ZeroGrad();
187 c.Backward();
188
189 ut::expect(CheckVectorEqual(c.Flatten(), TO_VEC(result_data)))
190 << c.Flatten() << " != " << TO_VEC(result_data) << "\n";
191 ut::expect(CheckVectorEqual(b->grad()->Flatten(), TO_VEC(b_grad_data)))
192 << "gradients: " << b->grad()->Flatten() << " != " << TO_VEC(b_grad_data) << "\n";
193 };
194
195 ut::test("TestTensorNorm") = [&ctx] {
196 std::vector<int> a_dims = {2, 2, 3};
197 float a_data[] = {0.7323f, -0.2345f, 0.4123f, 0.1234f, 0.2234f, -0.3371f,
198 0.7323f, -0.2345f, 0.4123f, 0.1234f, 0.2234f, -0.3371f};
199 float a_grad_data[] = {-0.268548f, -0.132720f, 0.401268f, -2.745161f, 2.255450f, 0.489711f,
200 -0.268548f, -0.132720f, 0.401268f, -2.745161f, 2.255450f, 0.489711f};
201
202 float norm_data[] = {1.066593f, -1.337468f, 0.270875f, 0.492263f, 0.901913f, -1.394176f,
203 1.066593f, -1.337468f, 0.270875f, 0.492263f, 0.901913f, -1.394176f};
204
205 std::vector<int> c_dims = {2, 3, 3};
206 float c_data[] = {0.134287f, 0.829944f, -0.215419f, -0.760618f, 0.586073f, 0.464571f,
207 0.665903f, -0.128526f, 0.454756f, 0.134287f, 0.829944f, -0.215419f,
208 -0.760618f, 0.586073f, 0.464571f, 0.665903f, -0.128526f, 0.454756f};
209
210 auto a = ctx.NewTensor(a_dims)->Fill(a_data);
211 auto &b = a->Norm();
212 auto c = ctx.NewTensor(c_dims)->Fill(c_data);
213 auto &d = b.MatMul(*c);
214
215 d.Forward();
216 d.ZeroGrad();
217 d.Backward();
218
219 ut::expect(CheckVectorEqual(b.Flatten(), TO_VEC(norm_data)))
220 << b.Flatten() << " != " << TO_VEC(norm_data) << "\n";
221 ut::expect(CheckVectorEqual(a->grad()->Flatten(), TO_VEC(a_grad_data)))
222 << "gradients: " << a->grad()->Flatten() << " != " << TO_VEC(a_grad_data) << "\n";
223 };
224
225 ut::test("TestTensorBroadcastAdd") = [&ctx] {
226 std::vector<int> a_dims = {2, 3, 2};
227 float a_data[] = {-0.725446f, 0.189742f, -0.337311f, -0.109581f, -0.344868f, 0.790175f,
228 -0.882165f, -0.972059f, -0.197083f, 0.185942f, -0.532996f, 0.035993f};
229 float a_grad_data[] = {1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f,
230 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f};
231 std::vector<int> b_dims = {2, 1, 2};
232 float b_data[] = {-0.845297f, -0.231948f, -0.285176f, -0.540625f};
233 float b_grad_data[] = {3.000000f, 3.000000f, 3.000000f, 3.000000f};
234 std::vector<int> result_dims = {2, 3, 2};
235 float result_data[] = {-1.570742f, -0.042206f, -1.182608f, -0.341529f, -1.190165f, 0.558227f,
236 -1.167341f, -1.512683f, -0.482260f, -0.354683f, -0.818172f, -0.504632f};
237
238 auto &a = *ctx.NewTensor(a_dims)->Fill(a_data);
239 auto &b = *ctx.NewTensor(b_dims)->Fill(b_data);
240 auto &c = a + b;
241 c.Forward();
242 c.ZeroGrad();
243 c.Backward();
244
245 ut::expect(CheckVectorEqual(c.Flatten(), TO_VEC(result_data)))
246 << c.Flatten() << " != " << TO_VEC(result_data) << "\n";
247 ut::expect(CheckVectorEqual(a.grad()->Flatten(), TO_VEC(a_grad_data)))
248 << "a grad: " << a.grad()->Flatten() << " != " << TO_VEC(a_grad_data) << "\n";
249 ut::expect(CheckVectorEqual(b.grad()->Flatten(), TO_VEC(b_grad_data)))
250 << "b grad: " << b.grad()->Flatten() << " != " << TO_VEC(b_grad_data) << "\n";
251 };
252
253 ut::test("TestTensorBroadcastMatMul") = [&ctx] {
254 std::vector<int> a_dims = {2, 3, 2};
255 float a_data[] = {-0.076818f, -0.239651f, -0.489204f, -0.388082f, -0.071079f, -0.728905f,
256 -0.950086f, 0.922597f, -0.275149f, -0.701350f, 0.798951f, -0.661272f};
257 float a_grad_data[] = {-1.944579f, 0.638576f, -1.944579f, 0.638576f, -1.944579f, 0.638576f,
258 -1.944579f, 0.638576f, -1.944579f, 0.638576f, -1.944579f, 0.638576f};
259 std::vector<int> b_dims = {1, 3, 2};
260 float b_data[] = {-0.060925f, 0.826868f, -0.963677f, -0.242371f, -0.919977f, 0.054079f};
261 float b_grad_data[] = {-1.063383f, -1.796663f, -1.063383f, -1.796663f, -1.063383f, -1.796663f};
262 std::vector<int> result_dims = {2, 3, 3};
263 float result_data[] = {-0.193480f, 0.132112f, 0.057710f, -0.291088f, 0.565494f, 0.429069f,
264 -0.598378f, 0.245162f, 0.025973f, 0.820750f, 0.691965f, 0.923950f,
265 -0.563161f, 0.435141f, 0.215202f, -0.595460f, -0.609658f, -0.770778f};
266
267 auto &a = *ctx.NewTensor(a_dims)->Fill(a_data);
268 auto &b = *ctx.NewTensor(b_dims)->Fill(b_data);
269 auto &c = a.MatMul(b);
270 c.Forward();
271 c.ZeroGrad();
272 c.Backward();
273 ut::expect(CheckVectorEqual(c.Flatten(), TO_VEC(result_data)))
274 << c.Flatten() << " != " << TO_VEC(result_data) << "\n";
275 ut::expect(CheckVectorEqual(a.grad()->Flatten(), TO_VEC(a_grad_data)))
276 << "a grad: " << a.grad()->Flatten() << " != " << TO_VEC(a_grad_data) << "\n";
277 ut::expect(CheckVectorEqual(b.grad()->Flatten(), TO_VEC(b_grad_data)))
278 << "b grad: " << b.grad()->Flatten() << " != " << TO_VEC(b_grad_data) << "\n";
279 };
280
281 ut::test("TestTensorTranspose") = [&ctx] {
282 auto x = ctx.NewTensor({2, 3, 4})->Fill<float>(
283 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23});
284 auto &x_t1 = x->Transpose(0, 1);
285 x_t1.Forward();
286 auto &x_t2 = x->Transpose(1, 2);
287 x_t2.Forward();
288
289 std::vector<float> x_t1_data = {0, 1, 2, 3, 12, 13, 14, 15, 4, 5, 6, 7,
290 16, 17, 18, 19, 8, 9, 10, 11, 20, 21, 22, 23};
291 ut::expect(CheckVectorEqual(x_t1.Flatten(), x_t1_data)) << x_t1.Flatten() << " != " << x_t1_data << "\n";
292
293 std::vector<float> x_t2_data = {0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11,
294 12, 16, 20, 13, 17, 21, 14, 18, 22, 15, 19, 23};
295
296 ut::expect(CheckVectorEqual(x_t2.Flatten(), x_t2_data)) << x_t2.Flatten() << " != " << x_t2_data << "\n";
297
298 auto x_t1_grad = x_t1.RandomGrad()->grad();
299 x_t1.Backward(false);
300 auto x_t1_grad_t1 = x_t1_grad->Transpose(0, 1);
301 x_t1_grad_t1.Forward();
302
303 ut::expect(CheckVectorEqual(x->grad()->Flatten(), x_t1_grad_t1.Flatten()))
304 << "x gradients: " << x->grad()->Flatten() << " != " << x_t1_grad_t1.Flatten() << "\n";
305 };
306
307 ut::test("TestTensorView1") = [&ctx] {
308 float x_data[] = {-0.141649f, -0.125871f, -1.265625f, -0.557144f, -0.593670f, -0.271154f,
309 2.106452f, -0.763804f, 0.276285f, -1.824517f, -0.027871f, 0.173058f,
310 -1.062135f, -1.543186f, -0.903289f, -1.008523f, -1.003833f, -0.068816f,
311 0.494010f, -0.588222f, 0.829179f, -0.457088f, 1.069247f, 0.996707f};
312 std::vector<int> x_dims = {2, 3, 4};
313 std::vector<int> y_dims = {2, 3, 2, 2};
314 std::vector<int> z_dims = {2, 1, 3, 4};
315 auto x = ctx.NewTensor(x_dims)->Fill(x_data);
316 auto &y = x->View(y_dims);
317 auto y_grad = y.RandomGrad()->grad();
318 auto &z = x->View(z_dims);
319 auto z_grad = z.RandomGrad()->grad();
320 auto x_grad = y_grad->View(x_dims) + z_grad->View(x_dims);
321 x_grad.Forward();
322 y.Forward();
323 y.Backward(false);
324 z.Forward();
325 z.Backward(false);
326 ut::expect(y.Dims() == y_dims) << "y shape: " << y.Dims() << " != " << y_dims << "\n";
327 ut::expect(CheckVectorEqual(y.Flatten(), TO_VEC(x_data))) << y.Flatten() << " != " << TO_VEC(x_data) << "\n";
328 ut::expect(z.Dims() == z_dims) << "z shape: " << z.Dims() << " != " << z_dims << "\n";
329 ut::expect(CheckVectorEqual(z.Flatten(), TO_VEC(x_data))) << z.Flatten() << " != " << TO_VEC(x_data) << "\n";
330 ut::expect(CheckVectorEqual(x->grad()->Flatten(), x_grad.Flatten()))
331 << "x grad: " << x->grad()->Flatten() << " != " << x_grad.Flatten() << "\n";
332 };
333
334 ut::test("TestTensorView2") = [&ctx] {
335 float x_data[] = {-0.141649f, -0.125871f, -1.265625f, -0.557144f, -0.593670f, -0.271154f,
336 2.106452f, -0.763804f, 0.276285f, -1.824517f, -0.027871f, 0.173058f,
337 -1.062135f, -1.543186f, -0.903289f, -1.008523f, -1.003833f, -0.068816f,
338 0.494010f, -0.588222f, 0.829179f, -0.457088f, 1.069247f, 0.996707f};
339 std::vector<int> x_dims = {4, 6};
340 std::vector<int> y_dims = {2, 2, 2, 3};
341 std::vector<int> z_dims = {1, 4, 1, 6};
342 auto x = ctx.NewTensor(x_dims)->Fill(x_data);
343 auto &y = x->View(y_dims);
344 auto y_grad = y.RandomGrad()->grad();
345 auto &z = x->View(z_dims);
346 auto z_grad = z.RandomGrad()->grad();
347 auto x_grad = y_grad->View(x_dims) + z_grad->View(x_dims);
348 x_grad.Forward();
349 y.Forward();
350 y.Backward(false);
351 z.Forward();
352 z.Backward(false);
353 ut::expect(y.Dims() == y_dims) << "y shape: " << y.Dims() << " != " << y_dims << "\n";
354 ut::expect(CheckVectorEqual(y.Flatten(), TO_VEC(x_data))) << y.Flatten() << " != " << TO_VEC(x_data) << "\n";
355 ut::expect(z.Dims() == z_dims) << "z shape: " << z.Dims() << " != " << z_dims << "\n";
356 ut::expect(CheckVectorEqual(z.Flatten(), TO_VEC(x_data))) << z.Flatten() << " != " << TO_VEC(x_data) << "\n";
357 ut::expect(CheckVectorEqual(x->grad()->Flatten(), x_grad.Flatten()))
358 << "x grad: " << x->grad()->Flatten() << " != " << x_grad.Flatten() << "\n";
359 };
360
361 ut::test("TestTensorView3") = [&ctx] {
362 float x_data[] = {-0.141649f, -0.125871f, -1.265625f, -0.557144f, -0.593670f, -0.271154f,
363 2.106452f, -0.763804f, 0.276285f, -1.824517f, -0.027871f, 0.173058f,
364 -1.062135f, -1.543186f, -0.903289f, -1.008523f, -1.003833f, -0.068816f,
365 0.494010f, -0.588222f, 0.829179f, -0.457088f, 1.069247f, 0.996707f};
366 std::vector<int> x_dims = {2, 2, 2, 3};
367 std::vector<int> y_dims = {4, 6};
368 std::vector<int> z_dims = {2, 4, 3};
369 auto x = ctx.NewTensor(x_dims)->Fill(x_data);
370 auto &y = x->View(y_dims);
371 auto y_grad = y.RandomGrad()->grad();
372 auto &z = x->View(z_dims);
373 auto z_grad = z.RandomGrad()->grad();
374 auto x_grad = y_grad->View(x_dims) + z_grad->View(x_dims);
375 x_grad.Forward();
376 y.Forward();
377 y.Backward(false);
378 z.Forward();
379 z.Backward(false);
380 ut::expect(y.Dims() == y_dims) << "y shape: " << y.Dims() << " != " << y_dims << "\n";
381 ut::expect(CheckVectorEqual(y.Flatten(), TO_VEC(x_data))) << y.Flatten() << " != " << TO_VEC(x_data) << "\n";
382 ut::expect(z.Dims() == z_dims) << "z shape: " << z.Dims() << " != " << z_dims << "\n";
383 ut::expect(CheckVectorEqual(z.Flatten(), TO_VEC(x_data))) << z.Flatten() << " != " << TO_VEC(x_data) << "\n";
384 ut::expect(CheckVectorEqual(x->grad()->Flatten(), x_grad.Flatten()))
385 << "x grad: " << x->grad()->Flatten() << " != " << x_grad.Flatten() << "\n";
386 };
387
388 ut::test("TestTensorView4") = [&ctx] {
389 float x_data[] = {-0.141649f, -0.125871f, -1.265625f, -0.557144f, -0.593670f, -0.271154f,
390 2.106452f, -0.763804f, 0.276285f, -1.824517f, -0.027871f, 0.173058f,
391
392 -1.062135f, -1.543186f, -0.903289f, -1.008523f, -1.003833f, -0.068816f,
393 0.494010f, -0.588222f, 0.829179f, -0.457088f, 1.069247f, 0.996707f};
394 std::vector<int> x_dims = {2, 2, 6};
395 std::vector<int> y_dims = {4, 2};
396 float y_data[] = {-1.265625f, -0.557144f, 0.276285f, -1.824517f, -0.903289f, -1.008523f, 0.829179f, -0.457088f};
397 std::vector<int> z_dims = {4, 1, 3};
398 float z_data[] = {-0.557144f, -0.593670f, -0.271154f, -1.824517f, -0.027871f, 0.173058f,
399 -1.008523f, -1.003833f, -0.068816f, -0.457088f, 1.069247f, 0.996707f};
400 auto x = ctx.NewTensor(x_dims)->Fill(x_data);
401 auto &y = x->View(y_dims, 1, 2);
402 auto &z = x->View(z_dims, 1, 2);
403 y.Forward();
404 z.Forward();
405 ut::expect(y.Dims() == y_dims) << "y shape: " << y.Dims() << " != " << y_dims << "\n";
406 ut::expect(CheckVectorEqual(y.Flatten(), TO_VEC(y_data))) << y.Flatten() << " != " << TO_VEC(y_data) << "\n";
407 ut::expect(z.Dims() == z_dims) << "z shape: " << z.Dims() << " != " << z_dims << "\n";
408 ut::expect(CheckVectorEqual(z.Flatten(), TO_VEC(z_data))) << z.Flatten() << " != " << TO_VEC(z_data) << "\n";
409
410 y.ZeroGrad();
411 auto y_grad = y.RandomGrad()->grad();
412 y.Backward(false);
413 auto &y_grad_truth = x->grad()->View(y_dims, 1, 2);
414 y_grad_truth.Forward();
415 ut::expect(CheckVectorEqual(y_grad->Flatten(), y_grad_truth.Flatten()))
416 << "y grad: " << y_grad->Flatten() << " != " << y_grad_truth.Flatten() << "\n";
417
418 z.ZeroGrad();
419 auto z_grad = z.RandomGrad()->grad();
420 z.Backward(false);
421 auto &z_grad_truth = x->grad()->View(z_dims, 1, 2);
422 z_grad_truth.Forward();
423 ut::expect(CheckVectorEqual(z_grad->Flatten(), z_grad_truth.Flatten()))
424 << "z grad: " << z_grad->Flatten() << " != " << z_grad_truth.Flatten() << "\n";
425 };
426
427 ut::test("TestTensorSplit") = [&ctx] {
428 float x_data[] = {-0.141649f, -0.125871f, -1.265625f, -0.557144f, -0.593670f, -0.271154f, 2.106452f,
429 -0.763804f, 0.276285f, -1.824517f, -0.027871f, 0.173058f, -1.062135f, -1.543186f,
430 -0.903289f, -1.008523f, -1.003833f, -0.068816f, 0.494010f, -0.588222f, 0.829179f,
431 -0.457088f, 1.069247f, 0.996707f, 0.069449f, -0.040386f, -0.372081f, 1.546610f,
432 0.060131f, -0.008539f, 1.380819f, -1.187713f, 0.851277f, 0.755909f, 0.320159f,
433 0.047182f, -1.942095f, -0.469703f, 0.206541f, -0.570447f, -0.161775f, -0.976735f,
434 0.450884f, 0.903816f, -0.139475f, -0.567350f, -0.927794f, -0.488965f};
435
436 auto x = ctx.NewTensor({2, 6, 4})->Fill(x_data);
437 auto splits = x->Split(2, 1);
438
439 std::vector<float> a_data = {-0.141649f, -0.125871f, -1.265625f, -0.557144f, -0.593670f, -0.271154f,
440 2.106452f, -0.763804f, 0.069449f, -0.040386f, -0.372081f, 1.546610f,
441 0.060131f, -0.008539f, 1.380819f, -1.187713f};
442 std::vector<float> b_data = {0.276285f, -1.824517f, -0.027871f, 0.173058f, -1.062135f, -1.543186f,
443 -0.903289f, -1.008523f, 0.851277f, 0.755909f, 0.320159f, 0.047182f,
444 -1.942095f, -0.469703f, 0.206541f, -0.570447f};
445 std::vector<float> c_data = {-1.003833f, -0.068816f, 0.494010f, -0.588222f, 0.829179f, -0.457088f,
446 1.069247f, 0.996707f, -0.161775f, -0.976735f, 0.450884f, 0.903816f,
447 -0.139475f, -0.567350f, -0.927794f, -0.488965f};
448 auto a = splits[0], b = splits[1], c = splits[2];
449 c->Forward();
450 ut::expect(CheckVectorEqual(c->Flatten(), c_data)) << c->Flatten() << " != " << c_data << "\n";
451
452 auto splits2 = x->Split(2, 2);
453
454 std::vector<float> e_data = {-0.141649f, -0.125871f, -0.593670f, -0.271154f, 0.276285f, -1.824517f,
455 -1.062135f, -1.543186f, -1.003833f, -0.068816f, 0.829179f, -0.457088f,
456 0.069449f, -0.040386f, 0.060131f, -0.008539f, 0.851277f, 0.755909f,
457 -1.942095f, -0.469703f, -0.161775f, -0.976735f, -0.139475f, -0.567350f};
458 std::vector<float> f_data = {-1.265625f, -0.557144f, 2.106452f, -0.763804f, -0.027871f, 0.173058f,
459 -0.903289f, -1.008523f, 0.494010f, -0.588222f, 1.069247f, 0.996707f,
460 -0.372081f, 1.546610f, 1.380819f, -1.187713f, 0.320159f, 0.047182f,
461 0.206541f, -0.570447f, 0.450884f, 0.903816f, -0.927794f, -0.488965f};
462
463 auto &d = a->MatMul(*b);
464 d.Forward();
465 d.Backward();
466 ut::expect(CheckVectorEqual(a->Flatten(), a_data)) << a->Flatten() << " != " << a_data << "\n";
467 ut::expect(CheckVectorEqual(b->Flatten(), b_data)) << b->Flatten() << " != " << b_data << "\n";
468 std::vector<float> d_data = {0.129375f, 2.049810f, 0.139811f, -0.083424f,
469 -0.017561f, -1.075018f, 0.430777f, 0.849955f};
470 ut::expect(CheckVectorEqual(d.Flatten(), d_data)) << d.Flatten() << " != " << d_data << "\n";
471
472 std::vector<float> x_grad_data = {
473 -0.785850f, -3.367703f, -0.931161f, -0.835465f, -0.785850f, -3.367703f, -0.931161f, -0.835465f,
474 -0.735318f, -0.397025f, 0.840827f, -1.320947f, -0.735318f, -0.397025f, 0.840827f, -1.320947f,
475 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f,
476 -1.090818f, 0.286207f, 0.526700f, -0.523265f, -1.090818f, 0.286207f, 0.526700f, -0.523265f,
477 0.129580f, -0.048925f, 1.008738f, 0.358897f, 0.129580f, -0.048925f, 1.008738f, 0.358897f,
478 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f};
479
480 ut::expect(CheckVectorEqual(x->grad()->Flatten(), x_grad_data))
481 << "x gradients: " << x->grad()->Flatten() << " != " << x_grad_data << "\n";
482
483 auto e = splits2[0], f = splits2[1];
484 e->Forward();
485 f->Forward();
486 ut::expect(CheckVectorEqual(e->Flatten(), e_data)) << e->Flatten() << " != " << e_data << "\n";
487 ut::expect(CheckVectorEqual(f->Flatten(), f_data)) << f->Flatten() << " != " << f_data << "\n";
488
489 auto &g = e->MatMul(*f);
490 g.Forward();
491 // forward is idempotent
492 g.Forward();
493 std::vector<float> g_data = {
494 0.249402f, -0.202235f, -0.017835f, 0.254894f, 0.004064f, -0.276914f, 0.902435f, -1.043429f, -0.030379f,
495 0.809721f, -0.133780f, -0.905041f, 0.666845f, 1.975553f, -0.323448f, 1.590503f, 1.209709f, -1.523092f,
496 2.204041f, -1.058645f, -0.237458f, 2.515754f, 0.383031f, -2.673788f, 1.308817f, -2.061964f, 0.016069f,
497 0.976155f, -0.455424f, -1.141936f, -0.794766f, 2.095752f, -0.102213f, -0.288005f, 0.678492f, 0.431015f,
498 -0.088302f, 0.143864f, 0.020329f, 0.037382f, -0.005188f, -0.044687f, -0.035580f, 0.093172f, 0.018849f,
499 0.017291f, 0.019394f, -0.051614f, 0.852353f, 0.277656f, 0.308209f, -0.255383f, 1.067030f, -1.159422f,
500 -0.003830f, -2.123810f, -0.643941f, -0.133182f, -1.300185f, 2.031532f, -1.450435f, 0.936698f, -0.097878f,
501 0.523763f, -0.955731f, 0.627684f, -0.825572f, 0.481258f, -0.071423f, 0.294836f, -0.575667f, 0.406818f};
502
503 ut::expect(CheckVectorEqual(g.Flatten(), g_data)) << g.Flatten() << " != " << g_data << "\n";
504 g.Backward();
505 std::vector<float> x_grad_data2 = {
506 0.687073f, -5.115632f, -2.626983f, -5.126099f, 0.687073f, -5.115632f, -2.626983f, -5.126099f,
507 0.737605f, -2.144953f, -0.854996f, -5.611580f, 0.737605f, -2.144953f, -0.854996f, -5.611580f,
508 1.472923f, -1.747928f, -1.695822f, -4.290633f, 1.472923f, -1.747928f, -1.695822f, -4.290633f,
509 -0.032290f, 0.536689f, -0.735788f, -1.830068f, -0.032290f, 0.536689f, -0.735788f, -1.830068f,
510 1.188109f, 0.201558f, -0.253751f, -0.947906f, 1.188109f, 0.201558f, -0.253751f, -0.947906f,
511 1.058529f, 0.250483f, -1.262489f, -1.306803f, 1.058529f, 0.250483f, -1.262489f, -1.306803f};
512 ut::expect(CheckVectorEqual(x->grad()->Flatten(), x_grad_data2))
513 << "x gradients: " << x->grad()->Flatten() << " != " << x_grad_data2 << "\n";
514 };
515
516 ut::test("TestTensorGelu") = [&ctx] {
517 std::vector<int> dims = {2, 3};
518 float x_data[] = {-1.119413f, 0.348449f, 0.720475f, -0.974813f, 1.244371f, 1.265503f};
519 float x_grad_data[] = {0.031505f, 0.340091f, 1.085720f, 0.024649f, 2.494036f, 2.551048f};
520 float y_data[] = {-0.147375f, 0.221696f, 0.550659f, -0.160819f, 1.111398f, 1.135126f};
521 float y_grad_data[] = {-0.294750f, 0.443392f, 1.101318f, -0.321638f, 2.222795f, 2.270252f};
522 auto x = ctx.NewTensor(dims)->Fill(x_data);
523 auto y = x->Gelu().FillGrad(y_grad_data);
524 y->Forward();
525 y->Backward(false);
526 ut::expect(CheckVectorEqual(y->Flatten(), TO_VEC(y_data))) << y->Flatten() << " != " << TO_VEC(y_data) << "\n";
527 ut::expect(CheckVectorEqual(x->grad()->Flatten(), TO_VEC(x_grad_data)))
528 << "x gradients: " << x->grad()->Flatten() << " != " << TO_VEC(x_grad_data) << "\n";
529 };
530
531 ut::test("TestTensorSoftmax") = [&ctx] {
532 std::vector<int> dims = {2, 2, 5};
533 float x_data[] = {0.111361f, -0.000386f, -1.804357f, 0.369323f, 1.306343f, -0.004050f, 1.898653f,
534 1.455873f, 0.272952f, 1.880325f, 0.164516f, -0.822872f, 1.178338f, -0.938773f,
535 -1.107920f, 0.836621f, 0.351749f, -0.460701f, 0.153248f, -0.459156f};
536 float x_grad_data[] = {-0.053102f, -0.051777f, -0.013515f, -0.051447f, 0.169841f, -0.022711f, 0.040656f,
537 -0.025984f, -0.027836f, 0.035875f, -0.078526f, -0.050058f, 0.214313f, -0.045784f,
538 -0.039946f, 0.093127f, -0.009149f, -0.030365f, -0.023234f, -0.030379f};
539
540 float y_data[] = {0.150618f, 0.134694f, 0.022176f, 0.194944f, 0.497568f, 0.050223f, 0.336697f,
541 0.216243f, 0.066253f, 0.330583f, 0.210944f, 0.078587f, 0.581386f, 0.069987f,
542 0.059096f, 0.374864f, 0.230832f, 0.102436f, 0.189273f, 0.102595f};
543
544 float y_grad_data[] = {0.301237f, 0.269387f, 0.044353f, 0.389888f, 0.995135f, 0.100447f, 0.673395f,
545 0.432487f, 0.132506f, 0.661165f, 0.421889f, 0.157174f, 1.162773f, 0.139973f,
546 0.118191f, 0.749728f, 0.461664f, 0.204873f, 0.378546f, 0.205189f};
547 auto x = ctx.NewTensor(dims)->Fill(x_data);
548 auto y = x->Softmax().FillGrad(y_grad_data);
549 y->Forward();
550 y->Backward(false);
551 ut::expect(CheckVectorEqual(y->Flatten(), TO_VEC(y_data))) << y->Flatten() << " != " << TO_VEC(y_data) << "\n";
552 ut::expect(CheckVectorEqual(x->grad()->Flatten(), TO_VEC(x_grad_data)))
553 << "x gradients: " << x->grad()->Flatten() << " != " << TO_VEC(x_grad_data) << "\n";
554 };
555
556 ut::test("TestTensorSoftmaxCasual") = [&ctx] {
557 std::vector<int> dims = {2, 2, 5};
558 float x_data[] = {0.111361f, -0.000386f, -1.804357f, 0.369323f, 1.306343f, -0.004050f, 1.898653f,
559 1.455873f, 0.272952f, 1.880325f, 0.164516f, -0.822872f, 1.178338f, -0.938773f,
560 -1.107920f, 0.836621f, 0.351749f, -0.460701f, 0.153248f, -0.459156f};
561 float y_grad_data[] = {0.301237f, 0.269387f, 0.044353f, 0.389888f, 0.995135f, 0.100447f, 0.673395f,
562 0.432487f, 0.132506f, 0.661165f, 0.421889f, 0.157174f, 1.162773f, 0.139973f,
563 0.118191f, 0.749728f, 0.461664f, 0.204873f, 0.378546f, 0.205189f};
564 auto x = ctx.NewTensor(dims)->Fill(x_data);
565 auto y = x->Softmax(true).FillGrad(y_grad_data);
566 y->Forward();
567 y->Backward(false);
568
569 float y_data[] = {1, 0, 0, 0, 0, 0.129803, 0.870197, 0, 0, 0, 1, 0, 0, 0, 0, 0.618898, 0.381102, 0, 0, 0};
570 float x_grad_data[] = {0, 0, 0, 0, 0, -0.0647168, 0.0647168, 0, 0, 0,
571 0, 0, 0, 0, 0, 0.0679438, -0.0679437, 0, 0, 0};
572
573 ut::expect(CheckVectorEqual(y->Flatten(), TO_VEC(y_data))) << y->Flatten() << " != " << TO_VEC(y_data) << "\n";
574 ut::expect(CheckVectorEqual(x->grad()->Flatten(), TO_VEC(x_grad_data)))
575 << "x gradients: " << x->grad()->Flatten() << " != " << TO_VEC(x_grad_data) << "\n";
576 };
577
578 ut::test("TestTensorCrossEntropy") = [&ctx] {
579 std::vector<int> x_dims = {2, 2, 5};
580 float x_data[] = {0.651805f, -0.589272f, 1.826531f, 2.116205f, -0.652173f, 1.867640f, -0.457756f,
581 -0.216584f, -1.390391f, 0.527376f, -1.137916f, -1.974276f, -0.499488f, -1.299625f,
582 -1.138694f, -1.578463f, -1.322330f, -2.240308f, 0.190388f, -0.643646f};
583 float x_grad_data[] = {-0.222596f, 0.007922f, 0.088714f, 0.118521f, 0.007439f, 0.164216f, -0.233949f,
584 0.020429f, 0.006316f, 0.042988f, 0.048293f, -0.229075f, 0.091444f, 0.041083f,
585 0.048256f, 0.022284f, 0.028790f, -0.238504f, 0.130677f, 0.056752f};
586 std::vector<int> y_dims = {2, 2};
587 int targets_data[] = {0, 1, 1, 2};
588 float y_data[] = {2.210769f, 2.745677f, 2.480527f, 3.079425f};
589 float y_grad_data[] = {0.250000f, 0.250000f, 0.250000f, 0.250000f};
590
591 auto x = ctx.NewTensor(x_dims)->Fill(x_data);
592 auto targets = ctx.NewTensor(y_dims, kI32)->Fill(targets_data);
593 auto y = x->Softmax().CrossEntropy(*targets).FillGrad(y_grad_data);
594
595 y->Forward();
596 y->Backward(false);
597 ut::expect(CheckVectorEqual(y->Flatten(), TO_VEC(y_data))) << y->Flatten() << " != " << TO_VEC(y_data) << "\n";
598 ut::expect(CheckVectorEqual(x->grad()->Flatten(), TO_VEC(x_grad_data)))
599 << "x gradients: " << x->grad()->Flatten() << " != " << TO_VEC(x_grad_data) << "\n";
600 };
601}
602
603int main() {
604 TensorContext ctx((size_t)1024 * 1024);
605
606 TestTensor(ctx);
607 ctx.PrintLayout();
608
609 return 0;
610}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t