History

initial release

Yusong Gao committed 2 years ago

Commit d0d49a9No parent (first commit)21 files · +8268 −0Browse files

Showing the changes to test_tensor.cpp (1 of 21 files). Show all

Unified · Split

test_tensor.cpp+610−0
@@ -0,0 +1,610 @@
1+#include <iostream>
2+
3+#include "boost/ut.hpp"
4+#include "tinytorch.hpp"
5+
6+using namespace tinytorch; // NOLINT
7+using namespace boost; // NOLINT
8+
9+namespace std {
10+
11+template <typename T>
12+ostream &operator<<(ostream &os, const vector<T> &vec) {
13+ os << "[";
14+ for (size_t i = 0; i < vec.size(); i++) {
15+ os << vec[i];
16+ if (i < vec.size() - 1) {
17+ os << ", ";
18+ }
19+ }
20+ os << "]";
21+ return os;
22+}
23+
24+}; // namespace std
25+
26+#define TO_VEC(a) std::vector<float>(a, a + sizeof(a) / sizeof(float))
27+
28+const float eps = 1e-5; // NOLINT
29+
30+bool CheckVectorEqual(const std::vector<float> &a, const std::vector<float> &b) {
31+ if (a.size() != b.size()) {
32+ return false;
33+ }
34+ for (size_t i = 0; i < a.size(); i++) {
35+ if (std::isnan(a[i]) || std::isnan(b[i]) || fabs(a[i] - b[i]) > eps) {
36+ return false;
37+ }
38+ }
39+ return true;
40+}
41+
42+void TestTensor(TensorContext &ctx) {
43+ ut::test("TestTensorAdd") = [&ctx] {
44+ std::vector<int> a_dims = {2, 3, 2};
45+ float a_data[] = {-0.156085f, 0.119778f, 0.761198f, 0.347593f, -0.474833f, -0.314762f,
46+ -0.510900f, -0.273722f, -0.890527f, 0.362633f, -0.958626f, 0.358971f};
47+ std::vector<int> b_dims = {2, 3, 2};
48+ float b_data[] = {0.642503f, 0.716670f, -0.840944f, -0.951772f, -0.858585f, 0.895501f,
49+ -0.205840f, -0.693812f, -0.889058f, 0.434394f, 0.890442f, 0.936250f};
50+ std::vector<int> result_dims = {2, 3, 2};
51+ float result_data[] = {0.486419f, 0.836448f, -0.079746f, -0.604179f, -1.333419f, 0.580739f,
52+ -0.716740f, -0.967534f, -1.779585f, 0.797027f, -0.068184f, 1.295222f};
53+
54+ auto &a = *ctx.NewTensor(a_dims)->Fill(a_data);
55+ auto &b = *ctx.NewTensor(b_dims)->Fill(b_data);
56+ auto &c = a + b;
57+ c.Forward();
58+ ut::expect(CheckVectorEqual(c.Flatten(), TO_VEC(result_data)))
59+ << c.Flatten() << " != " << TO_VEC(result_data) << "\n";
60+ };
61+
62+ ut::test("TestTensorMul") = [&ctx] {
63+ std::vector<int> a_dims = {2, 3, 2};
64+ float a_data[] = {0.321711f, 0.549407f, -0.011015f, -0.610029f, -0.721674f, -0.857888f,
65+ -0.992038f, 0.380269f, 0.134258f, -0.639243f, -0.338716f, 0.289208f};
66+ float a_grad_data[] = {0.110795f, -0.960914f, 0.314134f, -0.145329f, -0.071354f, -0.052769f,
67+ 0.418165f, -0.533604f, -0.073091f, 0.607927f, 0.679904f, 0.377298f};
68+ std::vector<int> b_dims = {2, 3, 2};
69+ float b_data[] = {0.110795f, -0.960914f, 0.314134f, -0.145329f, -0.071354f, -0.052769f,
70+ 0.418165f, -0.533604f, -0.073091f, 0.607927f, 0.679904f, 0.377298f};
71+ float b_grad_data[] = {0.321711f, 0.549407f, -0.011015f, -0.610029f, -0.721674f, -0.857888f,
72+ -0.992038f, 0.380269f, 0.134258f, -0.639243f, -0.338716f, 0.289208f};
73+ std::vector<int> result_dims = {2, 3, 2};
74+ float result_data[] = {0.035644f, -0.527933f, -0.003460f, 0.088655f, 0.051494f, 0.045270f,
75+ -0.414835f, -0.202913f, -0.009813f, -0.388613f, -0.230294f, 0.109118f};
76+ auto &a = *ctx.NewTensor(a_dims)->Fill(a_data);
77+ auto &b = *ctx.NewTensor(b_dims)->Fill(b_data);
78+ auto b_grad = b.RandomGrad()->grad();
79+ auto &b_grad_truth = *ctx.NewTensor(b_dims)->Fill(b_grad_data) + *b_grad;
80+ b_grad_truth.Forward();
81+ auto &c = a * b;
82+ c.Forward();
83+ c.Backward();
84+
85+ ut::expect(CheckVectorEqual(c.Flatten(), TO_VEC(result_data)))
86+ << c.Flatten() << " != " << TO_VEC(result_data) << "\n";
87+ ut::expect(CheckVectorEqual(a.grad()->Flatten(), TO_VEC(a_grad_data)))
88+ << "a grad: " << a.grad()->Flatten() << " != " << TO_VEC(a_grad_data) << "\n";
89+ ut::expect(CheckVectorEqual(b.grad()->Flatten(), b_grad_truth.Flatten()))
90+ << "b grad: " << b.grad()->Flatten() << " != " << b_grad_truth.Flatten() << "\n";
91+ };
92+
93+ ut::test("TestTensorMulMat") = [&ctx] {
94+ std::vector<int> a_dims = {2, 3, 2};
95+ float a_data[] = {0.038856f, -0.205084f, -0.576147f, -0.253080f, -0.973295f, 0.236236f,
96+ 0.515430f, 0.641762f, -0.492475f, -0.325701f, -0.156254f, 0.507203f};
97+ std::vector<int> b_dims = {2, 4, 2};
98+ float b_data[] = {-0.926039f, 0.038612f, 0.818189f, 0.535480f, 0.217580f, 0.744733f, 0.888028f, -0.513070f,
99+ -0.341174f, -0.588775f, -0.814142f, 0.241264f, -0.224577f, -0.357855f, 0.081625f, -0.727418f};
100+ std::vector<int> result_dims = {2, 3, 4};
101+ float result_data[] = {-0.043901f, -0.078027f, -0.144279f, 0.139727f, 0.523762f, -0.606916f,
102+ -0.313835f, -0.381786f, 0.910431f, -0.669840f, -0.035837f, -0.985519f,
103+ -0.553705f, -0.264799f, -0.345412f, -0.424757f, 0.359784f, 0.322365f,
104+ 0.227153f, 0.196722f, -0.245319f, 0.249583f, -0.146414f, -0.381703f};
105+ auto &a = *ctx.NewTensor(a_dims)->Fill(a_data);
106+ auto &b = *ctx.NewTensor(b_dims)->Fill(b_data);
107+ auto &c = a.MatMul(b);
108+ c.Forward();
109+ ut::expect(CheckVectorEqual(c.Flatten(), TO_VEC(result_data)))
110+ << c.Flatten() << " != " << TO_VEC(result_data) << "\n";
111+ };
112+
113+ ut::test("TestTensorLinear") = [&ctx] {
114+ std::vector<int> a_dims = {2, 3, 2};
115+ float a_data[] = {-0.124705f, 0.461944f, -0.035526f, 0.603655f, -0.565689f, 0.580007f,
116+ 0.739768f, 0.555825f, 0.353833f, -0.393867f, -0.692182f, 0.200951f};
117+ float a_grad_data[] = {-0.334757f, -0.358625f, -0.334757f, -0.358625f, -0.334757f, -0.358625f,
118+ 0.461304f, -0.141092f, 0.461304f, -0.141092f, 0.461304f, -0.141092f};
119+ std::vector<int> b_dims = {2, 4, 2};
120+ float b_data[] = {-0.337031f, -0.849364f, 0.880950f, 0.673869f, -0.806584f, 0.693106f,
121+ -0.072092f, -0.876236f, -0.607369f, 0.573827f, -0.175670f, -0.252041f,
122+ 0.433276f, -0.186126f, 0.811068f, -0.276751f};
123+ float b_grad_data[] = {-0.725921f, 1.645606f, -0.725921f, 1.645606f, -0.725921f, 1.645606f,
124+ -0.725921f, 1.645606f, 0.401419f, 0.362910f, 0.401419f, 0.362910f,
125+ 0.401419f, 0.362910f, 0.401419f, 0.362910f};
126+ std::vector<int> c_dims = {2, 3, 4};
127+ float c_data[] = {0.405725f, -0.488369f, 0.916681f, -0.544182f, 0.213521f, -0.491035f, 0.491596f, -0.358687f,
128+ 0.530066f, -0.208435f, -0.675905f, 0.453980f, -0.926804f, -0.315975f, 0.534358f, -0.848598f,
129+ -0.431841f, -0.352661f, 0.723182f, 0.930128f, -0.483508f, -0.560935f, 0.256884f, 0.824113f};
130+ float c_grad_data[] = {1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f,
131+ 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f,
132+ 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f};
133+ std::vector<int> result_dims = {2, 3, 4};
134+ float result_data[] = {0.055396f, -0.286938f, 1.337443f, -0.939964f, -0.287228f, -0.115548f,
135+ 0.938648f, -0.885070f, 0.228083f, -0.315930f, 0.182377f, -0.013461f,
136+ -1.057168f, -0.586021f, 0.751428f, -0.402421f, -0.872759f, -0.315548f,
137+ 0.949798f, 1.326114f, 0.052213f, -0.489987f, -0.080424f, 0.207093f};
138+ float result_grad_data[] = {1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f,
139+ 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f,
140+ 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f,
141+ 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f};
142+
143+ auto &a = *ctx.NewTensor(a_dims)->Fill(a_data);
144+ auto &b = *ctx.NewTensor(b_dims)->Fill(b_data);
145+ auto &c = *ctx.NewTensor(c_dims)->Fill(c_data);
146+ auto &d = a.MatMul(b) + c;
147+ d.Forward();
148+ d.ZeroGrad();
149+ d.Backward();
150+
151+ // std::cout << "Tensor a: " << std::endl;
152+ // a.PrintTensor();
153+ // std::cout << "Tensor b: " << std::endl;
154+ // b.PrintTensor();
155+ // std::cout << "Tensor c: " << std::endl;
156+ // c.PrintTensor();
157+ // std::cout << "Tensor d: " << std::endl;
158+ // d.PrintTensor();
159+
160+ ut::expect(CheckVectorEqual(d.Flatten(), TO_VEC(result_data)))
161+ << d.Flatten() << " != " << TO_VEC(result_data) << "\n";
162+ ut::expect(CheckVectorEqual(d.grad()->Flatten(), TO_VEC(result_grad_data)))
163+ << "result grad: " << d.grad()->Flatten() << " != " << TO_VEC(result_grad_data) << "\n";
164+ ut::expect(CheckVectorEqual(a.grad()->Flatten(), TO_VEC(a_grad_data)))
165+ << "a grad: " << a.grad()->Flatten() << " != " << TO_VEC(a_grad_data) << "\n";
166+ ut::expect(CheckVectorEqual(b.grad()->Flatten(), TO_VEC(b_grad_data)))
167+ << "b grad: " << b.grad()->Flatten() << " != " << TO_VEC(b_grad_data) << "\n";
168+ ut::expect(CheckVectorEqual(c.grad()->Flatten(), TO_VEC(c_grad_data)))
169+ << "c grad: " << c.grad()->Flatten() << " != " << TO_VEC(c_grad_data) << "\n";
170+ };
171+
172+ ut::test("TestTensorLookup") = [&ctx] {
173+ int32_t a_data[] = {2, 1, 2, 0, 3, 4};
174+ auto a = ctx.NewTensor({2, 3}, kI32)->Fill(a_data);
175+ float b_data[] = {0.642503f, 0.716670f, -0.840944f, -0.951772f, -0.858585f, 0.895501f,
176+ -0.205840f, -0.693812f, -0.889058f, 0.434394f, 0.890442f, 0.936250f};
177+ float b_grad_data[] = {1.0f, 1.0f, 1.0f, 1.0f, 2.0f, 2.0f, 1.0f, 1.0f, 1.0f, 1.0f, 0.0f, 0.0f};
178+ auto b = ctx.NewTensor({6, 2})->Fill(b_data);
179+
180+ float result_data[] = {-0.858585f, 0.895501f, -0.840944f, -0.951772f, -0.858585f, 0.895501f,
181+ 0.642503f, 0.716670f, -0.205840f, -0.693812f, -0.889058f, 0.434394f};
182+
183+ auto &c = (*b)[*a];
184+
185+ c.Forward();
186+ c.ZeroGrad();
187+ c.Backward();
188+
189+ ut::expect(CheckVectorEqual(c.Flatten(), TO_VEC(result_data)))
190+ << c.Flatten() << " != " << TO_VEC(result_data) << "\n";
191+ ut::expect(CheckVectorEqual(b->grad()->Flatten(), TO_VEC(b_grad_data)))
192+ << "gradients: " << b->grad()->Flatten() << " != " << TO_VEC(b_grad_data) << "\n";
193+ };
194+
195+ ut::test("TestTensorNorm") = [&ctx] {
196+ std::vector<int> a_dims = {2, 2, 3};
197+ float a_data[] = {0.7323f, -0.2345f, 0.4123f, 0.1234f, 0.2234f, -0.3371f,
198+ 0.7323f, -0.2345f, 0.4123f, 0.1234f, 0.2234f, -0.3371f};
199+ float a_grad_data[] = {-0.268548f, -0.132720f, 0.401268f, -2.745161f, 2.255450f, 0.489711f,
200+ -0.268548f, -0.132720f, 0.401268f, -2.745161f, 2.255450f, 0.489711f};
201+
202+ float norm_data[] = {1.066593f, -1.337468f, 0.270875f, 0.492263f, 0.901913f, -1.394176f,
203+ 1.066593f, -1.337468f, 0.270875f, 0.492263f, 0.901913f, -1.394176f};
204+
205+ std::vector<int> c_dims = {2, 3, 3};
206+ float c_data[] = {0.134287f, 0.829944f, -0.215419f, -0.760618f, 0.586073f, 0.464571f,
207+ 0.665903f, -0.128526f, 0.454756f, 0.134287f, 0.829944f, -0.215419f,
208+ -0.760618f, 0.586073f, 0.464571f, 0.665903f, -0.128526f, 0.454756f};
209+
210+ auto a = ctx.NewTensor(a_dims)->Fill(a_data);
211+ auto &b = a->Norm();
212+ auto c = ctx.NewTensor(c_dims)->Fill(c_data);
213+ auto &d = b.MatMul(*c);
214+
215+ d.Forward();
216+ d.ZeroGrad();
217+ d.Backward();
218+
219+ ut::expect(CheckVectorEqual(b.Flatten(), TO_VEC(norm_data)))
220+ << b.Flatten() << " != " << TO_VEC(norm_data) << "\n";
221+ ut::expect(CheckVectorEqual(a->grad()->Flatten(), TO_VEC(a_grad_data)))
222+ << "gradients: " << a->grad()->Flatten() << " != " << TO_VEC(a_grad_data) << "\n";
223+ };
224+
225+ ut::test("TestTensorBroadcastAdd") = [&ctx] {
226+ std::vector<int> a_dims = {2, 3, 2};
227+ float a_data[] = {-0.725446f, 0.189742f, -0.337311f, -0.109581f, -0.344868f, 0.790175f,
228+ -0.882165f, -0.972059f, -0.197083f, 0.185942f, -0.532996f, 0.035993f};
229+ float a_grad_data[] = {1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f,
230+ 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f};
231+ std::vector<int> b_dims = {2, 1, 2};
232+ float b_data[] = {-0.845297f, -0.231948f, -0.285176f, -0.540625f};
233+ float b_grad_data[] = {3.000000f, 3.000000f, 3.000000f, 3.000000f};
234+ std::vector<int> result_dims = {2, 3, 2};
235+ float result_data[] = {-1.570742f, -0.042206f, -1.182608f, -0.341529f, -1.190165f, 0.558227f,
236+ -1.167341f, -1.512683f, -0.482260f, -0.354683f, -0.818172f, -0.504632f};
237+
238+ auto &a = *ctx.NewTensor(a_dims)->Fill(a_data);
239+ auto &b = *ctx.NewTensor(b_dims)->Fill(b_data);
240+ auto &c = a + b;
241+ c.Forward();
242+ c.ZeroGrad();
243+ c.Backward();
244+
245+ ut::expect(CheckVectorEqual(c.Flatten(), TO_VEC(result_data)))
246+ << c.Flatten() << " != " << TO_VEC(result_data) << "\n";
247+ ut::expect(CheckVectorEqual(a.grad()->Flatten(), TO_VEC(a_grad_data)))
248+ << "a grad: " << a.grad()->Flatten() << " != " << TO_VEC(a_grad_data) << "\n";
249+ ut::expect(CheckVectorEqual(b.grad()->Flatten(), TO_VEC(b_grad_data)))
250+ << "b grad: " << b.grad()->Flatten() << " != " << TO_VEC(b_grad_data) << "\n";
251+ };
252+
253+ ut::test("TestTensorBroadcastMatMul") = [&ctx] {
254+ std::vector<int> a_dims = {2, 3, 2};
255+ float a_data[] = {-0.076818f, -0.239651f, -0.489204f, -0.388082f, -0.071079f, -0.728905f,
256+ -0.950086f, 0.922597f, -0.275149f, -0.701350f, 0.798951f, -0.661272f};
257+ float a_grad_data[] = {-1.944579f, 0.638576f, -1.944579f, 0.638576f, -1.944579f, 0.638576f,
258+ -1.944579f, 0.638576f, -1.944579f, 0.638576f, -1.944579f, 0.638576f};
259+ std::vector<int> b_dims = {1, 3, 2};
260+ float b_data[] = {-0.060925f, 0.826868f, -0.963677f, -0.242371f, -0.919977f, 0.054079f};
261+ float b_grad_data[] = {-1.063383f, -1.796663f, -1.063383f, -1.796663f, -1.063383f, -1.796663f};
262+ std::vector<int> result_dims = {2, 3, 3};
263+ float result_data[] = {-0.193480f, 0.132112f, 0.057710f, -0.291088f, 0.565494f, 0.429069f,
264+ -0.598378f, 0.245162f, 0.025973f, 0.820750f, 0.691965f, 0.923950f,
265+ -0.563161f, 0.435141f, 0.215202f, -0.595460f, -0.609658f, -0.770778f};
266+
267+ auto &a = *ctx.NewTensor(a_dims)->Fill(a_data);
268+ auto &b = *ctx.NewTensor(b_dims)->Fill(b_data);
269+ auto &c = a.MatMul(b);
270+ c.Forward();
271+ c.ZeroGrad();
272+ c.Backward();
273+ ut::expect(CheckVectorEqual(c.Flatten(), TO_VEC(result_data)))
274+ << c.Flatten() << " != " << TO_VEC(result_data) << "\n";
275+ ut::expect(CheckVectorEqual(a.grad()->Flatten(), TO_VEC(a_grad_data)))
276+ << "a grad: " << a.grad()->Flatten() << " != " << TO_VEC(a_grad_data) << "\n";
277+ ut::expect(CheckVectorEqual(b.grad()->Flatten(), TO_VEC(b_grad_data)))
278+ << "b grad: " << b.grad()->Flatten() << " != " << TO_VEC(b_grad_data) << "\n";
279+ };
280+
281+ ut::test("TestTensorTranspose") = [&ctx] {
282+ auto x = ctx.NewTensor({2, 3, 4})->Fill<float>(
283+ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23});
284+ auto &x_t1 = x->Transpose(0, 1);
285+ x_t1.Forward();
286+ auto &x_t2 = x->Transpose(1, 2);
287+ x_t2.Forward();
288+
289+ std::vector<float> x_t1_data = {0, 1, 2, 3, 12, 13, 14, 15, 4, 5, 6, 7,
290+ 16, 17, 18, 19, 8, 9, 10, 11, 20, 21, 22, 23};
291+ ut::expect(CheckVectorEqual(x_t1.Flatten(), x_t1_data)) << x_t1.Flatten() << " != " << x_t1_data << "\n";
292+
293+ std::vector<float> x_t2_data = {0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11,
294+ 12, 16, 20, 13, 17, 21, 14, 18, 22, 15, 19, 23};
295+
296+ ut::expect(CheckVectorEqual(x_t2.Flatten(), x_t2_data)) << x_t2.Flatten() << " != " << x_t2_data << "\n";
297+
298+ auto x_t1_grad = x_t1.RandomGrad()->grad();
299+ x_t1.Backward(false);
300+ auto x_t1_grad_t1 = x_t1_grad->Transpose(0, 1);
301+ x_t1_grad_t1.Forward();
302+
303+ ut::expect(CheckVectorEqual(x->grad()->Flatten(), x_t1_grad_t1.Flatten()))
304+ << "x gradients: " << x->grad()->Flatten() << " != " << x_t1_grad_t1.Flatten() << "\n";
305+ };
306+
307+ ut::test("TestTensorView1") = [&ctx] {
308+ float x_data[] = {-0.141649f, -0.125871f, -1.265625f, -0.557144f, -0.593670f, -0.271154f,
309+ 2.106452f, -0.763804f, 0.276285f, -1.824517f, -0.027871f, 0.173058f,
310+ -1.062135f, -1.543186f, -0.903289f, -1.008523f, -1.003833f, -0.068816f,
311+ 0.494010f, -0.588222f, 0.829179f, -0.457088f, 1.069247f, 0.996707f};
312+ std::vector<int> x_dims = {2, 3, 4};
313+ std::vector<int> y_dims = {2, 3, 2, 2};
314+ std::vector<int> z_dims = {2, 1, 3, 4};
315+ auto x = ctx.NewTensor(x_dims)->Fill(x_data);
316+ auto &y = x->View(y_dims);
317+ auto y_grad = y.RandomGrad()->grad();
318+ auto &z = x->View(z_dims);
319+ auto z_grad = z.RandomGrad()->grad();
320+ auto x_grad = y_grad->View(x_dims) + z_grad->View(x_dims);
321+ x_grad.Forward();
322+ y.Forward();
323+ y.Backward(false);
324+ z.Forward();
325+ z.Backward(false);
326+ ut::expect(y.Dims() == y_dims) << "y shape: " << y.Dims() << " != " << y_dims << "\n";
327+ ut::expect(CheckVectorEqual(y.Flatten(), TO_VEC(x_data))) << y.Flatten() << " != " << TO_VEC(x_data) << "\n";
328+ ut::expect(z.Dims() == z_dims) << "z shape: " << z.Dims() << " != " << z_dims << "\n";
329+ ut::expect(CheckVectorEqual(z.Flatten(), TO_VEC(x_data))) << z.Flatten() << " != " << TO_VEC(x_data) << "\n";
330+ ut::expect(CheckVectorEqual(x->grad()->Flatten(), x_grad.Flatten()))
331+ << "x grad: " << x->grad()->Flatten() << " != " << x_grad.Flatten() << "\n";
332+ };
333+
334+ ut::test("TestTensorView2") = [&ctx] {
335+ float x_data[] = {-0.141649f, -0.125871f, -1.265625f, -0.557144f, -0.593670f, -0.271154f,
336+ 2.106452f, -0.763804f, 0.276285f, -1.824517f, -0.027871f, 0.173058f,
337+ -1.062135f, -1.543186f, -0.903289f, -1.008523f, -1.003833f, -0.068816f,
338+ 0.494010f, -0.588222f, 0.829179f, -0.457088f, 1.069247f, 0.996707f};
339+ std::vector<int> x_dims = {4, 6};
340+ std::vector<int> y_dims = {2, 2, 2, 3};
341+ std::vector<int> z_dims = {1, 4, 1, 6};
342+ auto x = ctx.NewTensor(x_dims)->Fill(x_data);
343+ auto &y = x->View(y_dims);
344+ auto y_grad = y.RandomGrad()->grad();
345+ auto &z = x->View(z_dims);
346+ auto z_grad = z.RandomGrad()->grad();
347+ auto x_grad = y_grad->View(x_dims) + z_grad->View(x_dims);
348+ x_grad.Forward();
349+ y.Forward();
350+ y.Backward(false);
351+ z.Forward();
352+ z.Backward(false);
353+ ut::expect(y.Dims() == y_dims) << "y shape: " << y.Dims() << " != " << y_dims << "\n";
354+ ut::expect(CheckVectorEqual(y.Flatten(), TO_VEC(x_data))) << y.Flatten() << " != " << TO_VEC(x_data) << "\n";
355+ ut::expect(z.Dims() == z_dims) << "z shape: " << z.Dims() << " != " << z_dims << "\n";
356+ ut::expect(CheckVectorEqual(z.Flatten(), TO_VEC(x_data))) << z.Flatten() << " != " << TO_VEC(x_data) << "\n";
357+ ut::expect(CheckVectorEqual(x->grad()->Flatten(), x_grad.Flatten()))
358+ << "x grad: " << x->grad()->Flatten() << " != " << x_grad.Flatten() << "\n";
359+ };
360+
361+ ut::test("TestTensorView3") = [&ctx] {
362+ float x_data[] = {-0.141649f, -0.125871f, -1.265625f, -0.557144f, -0.593670f, -0.271154f,
363+ 2.106452f, -0.763804f, 0.276285f, -1.824517f, -0.027871f, 0.173058f,
364+ -1.062135f, -1.543186f, -0.903289f, -1.008523f, -1.003833f, -0.068816f,
365+ 0.494010f, -0.588222f, 0.829179f, -0.457088f, 1.069247f, 0.996707f};
366+ std::vector<int> x_dims = {2, 2, 2, 3};
367+ std::vector<int> y_dims = {4, 6};
368+ std::vector<int> z_dims = {2, 4, 3};
369+ auto x = ctx.NewTensor(x_dims)->Fill(x_data);
370+ auto &y = x->View(y_dims);
371+ auto y_grad = y.RandomGrad()->grad();
372+ auto &z = x->View(z_dims);
373+ auto z_grad = z.RandomGrad()->grad();
374+ auto x_grad = y_grad->View(x_dims) + z_grad->View(x_dims);
375+ x_grad.Forward();
376+ y.Forward();
377+ y.Backward(false);
378+ z.Forward();
379+ z.Backward(false);
380+ ut::expect(y.Dims() == y_dims) << "y shape: " << y.Dims() << " != " << y_dims << "\n";
381+ ut::expect(CheckVectorEqual(y.Flatten(), TO_VEC(x_data))) << y.Flatten() << " != " << TO_VEC(x_data) << "\n";
382+ ut::expect(z.Dims() == z_dims) << "z shape: " << z.Dims() << " != " << z_dims << "\n";
383+ ut::expect(CheckVectorEqual(z.Flatten(), TO_VEC(x_data))) << z.Flatten() << " != " << TO_VEC(x_data) << "\n";
384+ ut::expect(CheckVectorEqual(x->grad()->Flatten(), x_grad.Flatten()))
385+ << "x grad: " << x->grad()->Flatten() << " != " << x_grad.Flatten() << "\n";
386+ };
387+
388+ ut::test("TestTensorView4") = [&ctx] {
389+ float x_data[] = {-0.141649f, -0.125871f, -1.265625f, -0.557144f, -0.593670f, -0.271154f,
390+ 2.106452f, -0.763804f, 0.276285f, -1.824517f, -0.027871f, 0.173058f,
391+
392+ -1.062135f, -1.543186f, -0.903289f, -1.008523f, -1.003833f, -0.068816f,
393+ 0.494010f, -0.588222f, 0.829179f, -0.457088f, 1.069247f, 0.996707f};
394+ std::vector<int> x_dims = {2, 2, 6};
395+ std::vector<int> y_dims = {4, 2};
396+ float y_data[] = {-1.265625f, -0.557144f, 0.276285f, -1.824517f, -0.903289f, -1.008523f, 0.829179f, -0.457088f};
397+ std::vector<int> z_dims = {4, 1, 3};
398+ float z_data[] = {-0.557144f, -0.593670f, -0.271154f, -1.824517f, -0.027871f, 0.173058f,
399+ -1.008523f, -1.003833f, -0.068816f, -0.457088f, 1.069247f, 0.996707f};
400+ auto x = ctx.NewTensor(x_dims)->Fill(x_data);
401+ auto &y = x->View(y_dims, 1, 2);
402+ auto &z = x->View(z_dims, 1, 2);
403+ y.Forward();
404+ z.Forward();
405+ ut::expect(y.Dims() == y_dims) << "y shape: " << y.Dims() << " != " << y_dims << "\n";
406+ ut::expect(CheckVectorEqual(y.Flatten(), TO_VEC(y_data))) << y.Flatten() << " != " << TO_VEC(y_data) << "\n";
407+ ut::expect(z.Dims() == z_dims) << "z shape: " << z.Dims() << " != " << z_dims << "\n";
408+ ut::expect(CheckVectorEqual(z.Flatten(), TO_VEC(z_data))) << z.Flatten() << " != " << TO_VEC(z_data) << "\n";
409+
410+ y.ZeroGrad();
411+ auto y_grad = y.RandomGrad()->grad();
412+ y.Backward(false);
413+ auto &y_grad_truth = x->grad()->View(y_dims, 1, 2);
414+ y_grad_truth.Forward();
415+ ut::expect(CheckVectorEqual(y_grad->Flatten(), y_grad_truth.Flatten()))
416+ << "y grad: " << y_grad->Flatten() << " != " << y_grad_truth.Flatten() << "\n";
417+
418+ z.ZeroGrad();
419+ auto z_grad = z.RandomGrad()->grad();
420+ z.Backward(false);
421+ auto &z_grad_truth = x->grad()->View(z_dims, 1, 2);
422+ z_grad_truth.Forward();
423+ ut::expect(CheckVectorEqual(z_grad->Flatten(), z_grad_truth.Flatten()))
424+ << "z grad: " << z_grad->Flatten() << " != " << z_grad_truth.Flatten() << "\n";
425+ };
426+
427+ ut::test("TestTensorSplit") = [&ctx] {
428+ float x_data[] = {-0.141649f, -0.125871f, -1.265625f, -0.557144f, -0.593670f, -0.271154f, 2.106452f,
429+ -0.763804f, 0.276285f, -1.824517f, -0.027871f, 0.173058f, -1.062135f, -1.543186f,
430+ -0.903289f, -1.008523f, -1.003833f, -0.068816f, 0.494010f, -0.588222f, 0.829179f,
431+ -0.457088f, 1.069247f, 0.996707f, 0.069449f, -0.040386f, -0.372081f, 1.546610f,
432+ 0.060131f, -0.008539f, 1.380819f, -1.187713f, 0.851277f, 0.755909f, 0.320159f,
433+ 0.047182f, -1.942095f, -0.469703f, 0.206541f, -0.570447f, -0.161775f, -0.976735f,
434+ 0.450884f, 0.903816f, -0.139475f, -0.567350f, -0.927794f, -0.488965f};
435+
436+ auto x = ctx.NewTensor({2, 6, 4})->Fill(x_data);
437+ auto splits = x->Split(2, 1);
438+
439+ std::vector<float> a_data = {-0.141649f, -0.125871f, -1.265625f, -0.557144f, -0.593670f, -0.271154f,
440+ 2.106452f, -0.763804f, 0.069449f, -0.040386f, -0.372081f, 1.546610f,
441+ 0.060131f, -0.008539f, 1.380819f, -1.187713f};
442+ std::vector<float> b_data = {0.276285f, -1.824517f, -0.027871f, 0.173058f, -1.062135f, -1.543186f,
443+ -0.903289f, -1.008523f, 0.851277f, 0.755909f, 0.320159f, 0.047182f,
444+ -1.942095f, -0.469703f, 0.206541f, -0.570447f};
445+ std::vector<float> c_data = {-1.003833f, -0.068816f, 0.494010f, -0.588222f, 0.829179f, -0.457088f,
446+ 1.069247f, 0.996707f, -0.161775f, -0.976735f, 0.450884f, 0.903816f,
447+ -0.139475f, -0.567350f, -0.927794f, -0.488965f};
448+ auto a = splits[0], b = splits[1], c = splits[2];
449+ c->Forward();
450+ ut::expect(CheckVectorEqual(c->Flatten(), c_data)) << c->Flatten() << " != " << c_data << "\n";
451+
452+ auto splits2 = x->Split(2, 2);
453+
454+ std::vector<float> e_data = {-0.141649f, -0.125871f, -0.593670f, -0.271154f, 0.276285f, -1.824517f,
455+ -1.062135f, -1.543186f, -1.003833f, -0.068816f, 0.829179f, -0.457088f,
456+ 0.069449f, -0.040386f, 0.060131f, -0.008539f, 0.851277f, 0.755909f,
457+ -1.942095f, -0.469703f, -0.161775f, -0.976735f, -0.139475f, -0.567350f};
458+ std::vector<float> f_data = {-1.265625f, -0.557144f, 2.106452f, -0.763804f, -0.027871f, 0.173058f,
459+ -0.903289f, -1.008523f, 0.494010f, -0.588222f, 1.069247f, 0.996707f,
460+ -0.372081f, 1.546610f, 1.380819f, -1.187713f, 0.320159f, 0.047182f,
461+ 0.206541f, -0.570447f, 0.450884f, 0.903816f, -0.927794f, -0.488965f};
462+
463+ auto &d = a->MatMul(*b);
464+ d.Forward();
465+ d.Backward();
466+ ut::expect(CheckVectorEqual(a->Flatten(), a_data)) << a->Flatten() << " != " << a_data << "\n";
467+ ut::expect(CheckVectorEqual(b->Flatten(), b_data)) << b->Flatten() << " != " << b_data << "\n";
468+ std::vector<float> d_data = {0.129375f, 2.049810f, 0.139811f, -0.083424f,
469+ -0.017561f, -1.075018f, 0.430777f, 0.849955f};
470+ ut::expect(CheckVectorEqual(d.Flatten(), d_data)) << d.Flatten() << " != " << d_data << "\n";
471+
472+ std::vector<float> x_grad_data = {
473+ -0.785850f, -3.367703f, -0.931161f, -0.835465f, -0.785850f, -3.367703f, -0.931161f, -0.835465f,
474+ -0.735318f, -0.397025f, 0.840827f, -1.320947f, -0.735318f, -0.397025f, 0.840827f, -1.320947f,
475+ 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f,
476+ -1.090818f, 0.286207f, 0.526700f, -0.523265f, -1.090818f, 0.286207f, 0.526700f, -0.523265f,
477+ 0.129580f, -0.048925f, 1.008738f, 0.358897f, 0.129580f, -0.048925f, 1.008738f, 0.358897f,
478+ 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f};
479+
480+ ut::expect(CheckVectorEqual(x->grad()->Flatten(), x_grad_data))
481+ << "x gradients: " << x->grad()->Flatten() << " != " << x_grad_data << "\n";
482+
483+ auto e = splits2[0], f = splits2[1];
484+ e->Forward();
485+ f->Forward();
486+ ut::expect(CheckVectorEqual(e->Flatten(), e_data)) << e->Flatten() << " != " << e_data << "\n";
487+ ut::expect(CheckVectorEqual(f->Flatten(), f_data)) << f->Flatten() << " != " << f_data << "\n";
488+
489+ auto &g = e->MatMul(*f);
490+ g.Forward();
491+ // forward is idempotent
492+ g.Forward();
493+ std::vector<float> g_data = {
494+ 0.249402f, -0.202235f, -0.017835f, 0.254894f, 0.004064f, -0.276914f, 0.902435f, -1.043429f, -0.030379f,
495+ 0.809721f, -0.133780f, -0.905041f, 0.666845f, 1.975553f, -0.323448f, 1.590503f, 1.209709f, -1.523092f,
496+ 2.204041f, -1.058645f, -0.237458f, 2.515754f, 0.383031f, -2.673788f, 1.308817f, -2.061964f, 0.016069f,
497+ 0.976155f, -0.455424f, -1.141936f, -0.794766f, 2.095752f, -0.102213f, -0.288005f, 0.678492f, 0.431015f,
498+ -0.088302f, 0.143864f, 0.020329f, 0.037382f, -0.005188f, -0.044687f, -0.035580f, 0.093172f, 0.018849f,
499+ 0.017291f, 0.019394f, -0.051614f, 0.852353f, 0.277656f, 0.308209f, -0.255383f, 1.067030f, -1.159422f,
500+ -0.003830f, -2.123810f, -0.643941f, -0.133182f, -1.300185f, 2.031532f, -1.450435f, 0.936698f, -0.097878f,
501+ 0.523763f, -0.955731f, 0.627684f, -0.825572f, 0.481258f, -0.071423f, 0.294836f, -0.575667f, 0.406818f};
502+
503+ ut::expect(CheckVectorEqual(g.Flatten(), g_data)) << g.Flatten() << " != " << g_data << "\n";
504+ g.Backward();
505+ std::vector<float> x_grad_data2 = {
506+ 0.687073f, -5.115632f, -2.626983f, -5.126099f, 0.687073f, -5.115632f, -2.626983f, -5.126099f,
507+ 0.737605f, -2.144953f, -0.854996f, -5.611580f, 0.737605f, -2.144953f, -0.854996f, -5.611580f,
508+ 1.472923f, -1.747928f, -1.695822f, -4.290633f, 1.472923f, -1.747928f, -1.695822f, -4.290633f,
509+ -0.032290f, 0.536689f, -0.735788f, -1.830068f, -0.032290f, 0.536689f, -0.735788f, -1.830068f,
510+ 1.188109f, 0.201558f, -0.253751f, -0.947906f, 1.188109f, 0.201558f, -0.253751f, -0.947906f,
511+ 1.058529f, 0.250483f, -1.262489f, -1.306803f, 1.058529f, 0.250483f, -1.262489f, -1.306803f};
512+ ut::expect(CheckVectorEqual(x->grad()->Flatten(), x_grad_data2))
513+ << "x gradients: " << x->grad()->Flatten() << " != " << x_grad_data2 << "\n";
514+ };
515+
516+ ut::test("TestTensorGelu") = [&ctx] {
517+ std::vector<int> dims = {2, 3};
518+ float x_data[] = {-1.119413f, 0.348449f, 0.720475f, -0.974813f, 1.244371f, 1.265503f};
519+ float x_grad_data[] = {0.031505f, 0.340091f, 1.085720f, 0.024649f, 2.494036f, 2.551048f};
520+ float y_data[] = {-0.147375f, 0.221696f, 0.550659f, -0.160819f, 1.111398f, 1.135126f};
521+ float y_grad_data[] = {-0.294750f, 0.443392f, 1.101318f, -0.321638f, 2.222795f, 2.270252f};
522+ auto x = ctx.NewTensor(dims)->Fill(x_data);
523+ auto y = x->Gelu().FillGrad(y_grad_data);
524+ y->Forward();
525+ y->Backward(false);
526+ ut::expect(CheckVectorEqual(y->Flatten(), TO_VEC(y_data))) << y->Flatten() << " != " << TO_VEC(y_data) << "\n";
527+ ut::expect(CheckVectorEqual(x->grad()->Flatten(), TO_VEC(x_grad_data)))
528+ << "x gradients: " << x->grad()->Flatten() << " != " << TO_VEC(x_grad_data) << "\n";
529+ };
530+
531+ ut::test("TestTensorSoftmax") = [&ctx] {
532+ std::vector<int> dims = {2, 2, 5};
533+ float x_data[] = {0.111361f, -0.000386f, -1.804357f, 0.369323f, 1.306343f, -0.004050f, 1.898653f,
534+ 1.455873f, 0.272952f, 1.880325f, 0.164516f, -0.822872f, 1.178338f, -0.938773f,
535+ -1.107920f, 0.836621f, 0.351749f, -0.460701f, 0.153248f, -0.459156f};
536+ float x_grad_data[] = {-0.053102f, -0.051777f, -0.013515f, -0.051447f, 0.169841f, -0.022711f, 0.040656f,
537+ -0.025984f, -0.027836f, 0.035875f, -0.078526f, -0.050058f, 0.214313f, -0.045784f,
538+ -0.039946f, 0.093127f, -0.009149f, -0.030365f, -0.023234f, -0.030379f};
539+
540+ float y_data[] = {0.150618f, 0.134694f, 0.022176f, 0.194944f, 0.497568f, 0.050223f, 0.336697f,
541+ 0.216243f, 0.066253f, 0.330583f, 0.210944f, 0.078587f, 0.581386f, 0.069987f,
542+ 0.059096f, 0.374864f, 0.230832f, 0.102436f, 0.189273f, 0.102595f};
543+
544+ float y_grad_data[] = {0.301237f, 0.269387f, 0.044353f, 0.389888f, 0.995135f, 0.100447f, 0.673395f,
545+ 0.432487f, 0.132506f, 0.661165f, 0.421889f, 0.157174f, 1.162773f, 0.139973f,
546+ 0.118191f, 0.749728f, 0.461664f, 0.204873f, 0.378546f, 0.205189f};
547+ auto x = ctx.NewTensor(dims)->Fill(x_data);
548+ auto y = x->Softmax().FillGrad(y_grad_data);
549+ y->Forward();
550+ y->Backward(false);
551+ ut::expect(CheckVectorEqual(y->Flatten(), TO_VEC(y_data))) << y->Flatten() << " != " << TO_VEC(y_data) << "\n";
552+ ut::expect(CheckVectorEqual(x->grad()->Flatten(), TO_VEC(x_grad_data)))
553+ << "x gradients: " << x->grad()->Flatten() << " != " << TO_VEC(x_grad_data) << "\n";
554+ };
555+
556+ ut::test("TestTensorSoftmaxCasual") = [&ctx] {
557+ std::vector<int> dims = {2, 2, 5};
558+ float x_data[] = {0.111361f, -0.000386f, -1.804357f, 0.369323f, 1.306343f, -0.004050f, 1.898653f,
559+ 1.455873f, 0.272952f, 1.880325f, 0.164516f, -0.822872f, 1.178338f, -0.938773f,
560+ -1.107920f, 0.836621f, 0.351749f, -0.460701f, 0.153248f, -0.459156f};
561+ float y_grad_data[] = {0.301237f, 0.269387f, 0.044353f, 0.389888f, 0.995135f, 0.100447f, 0.673395f,
562+ 0.432487f, 0.132506f, 0.661165f, 0.421889f, 0.157174f, 1.162773f, 0.139973f,
563+ 0.118191f, 0.749728f, 0.461664f, 0.204873f, 0.378546f, 0.205189f};
564+ auto x = ctx.NewTensor(dims)->Fill(x_data);
565+ auto y = x->Softmax(true).FillGrad(y_grad_data);
566+ y->Forward();
567+ y->Backward(false);
568+
569+ float y_data[] = {1, 0, 0, 0, 0, 0.129803, 0.870197, 0, 0, 0, 1, 0, 0, 0, 0, 0.618898, 0.381102, 0, 0, 0};
570+ float x_grad_data[] = {0, 0, 0, 0, 0, -0.0647168, 0.0647168, 0, 0, 0,
571+ 0, 0, 0, 0, 0, 0.0679438, -0.0679437, 0, 0, 0};
572+
573+ ut::expect(CheckVectorEqual(y->Flatten(), TO_VEC(y_data))) << y->Flatten() << " != " << TO_VEC(y_data) << "\n";
574+ ut::expect(CheckVectorEqual(x->grad()->Flatten(), TO_VEC(x_grad_data)))
575+ << "x gradients: " << x->grad()->Flatten() << " != " << TO_VEC(x_grad_data) << "\n";
576+ };
577+
578+ ut::test("TestTensorCrossEntropy") = [&ctx] {
579+ std::vector<int> x_dims = {2, 2, 5};
580+ float x_data[] = {0.651805f, -0.589272f, 1.826531f, 2.116205f, -0.652173f, 1.867640f, -0.457756f,
581+ -0.216584f, -1.390391f, 0.527376f, -1.137916f, -1.974276f, -0.499488f, -1.299625f,
582+ -1.138694f, -1.578463f, -1.322330f, -2.240308f, 0.190388f, -0.643646f};
583+ float x_grad_data[] = {-0.222596f, 0.007922f, 0.088714f, 0.118521f, 0.007439f, 0.164216f, -0.233949f,
584+ 0.020429f, 0.006316f, 0.042988f, 0.048293f, -0.229075f, 0.091444f, 0.041083f,
585+ 0.048256f, 0.022284f, 0.028790f, -0.238504f, 0.130677f, 0.056752f};
586+ std::vector<int> y_dims = {2, 2};
587+ int targets_data[] = {0, 1, 1, 2};
588+ float y_data[] = {2.210769f, 2.745677f, 2.480527f, 3.079425f};
589+ float y_grad_data[] = {0.250000f, 0.250000f, 0.250000f, 0.250000f};
590+
591+ auto x = ctx.NewTensor(x_dims)->Fill(x_data);
592+ auto targets = ctx.NewTensor(y_dims, kI32)->Fill(targets_data);
593+ auto y = x->Softmax().CrossEntropy(*targets).FillGrad(y_grad_data);
594+
595+ y->Forward();
596+ y->Backward(false);
597+ ut::expect(CheckVectorEqual(y->Flatten(), TO_VEC(y_data))) << y->Flatten() << " != " << TO_VEC(y_data) << "\n";
598+ ut::expect(CheckVectorEqual(x->grad()->Flatten(), TO_VEC(x_grad_data)))
599+ << "x gradients: " << x->grad()->Flatten() << " != " << TO_VEC(x_grad_data) << "\n";
600+ };
601+}
602+
603+int main() {
604+ TensorContext ctx((size_t)1024 * 1024);
605+
606+ TestTensor(ctx);
607+ ctx.PrintLayout();
608+
609+ return 0;
610+}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t