/
rnekrasov
/
google-research
GigaCode
beta
ОбзорЦентр заботыВойти

google-research

Форк
0
Ветки: 311Коммиты: 4490Теги: 0
google-research
/automl_zero
/
generator_test.cc 
340 строк · 13.3 Кб
1
// Copyright 2024 The Google Research Authors.
2
//
3
// Licensed under the Apache License, Version 2.0 (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//     http://www.apache.org/licenses/LICENSE-2.0
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14


15
#include "generator.h"
16


17
#include <functional>
18
#include <limits>
19
#include <random>
20
#include <sstream>
21


22
#include "algorithm_test_util.h"
23
#include "task.h"
24
#include "task_util.h"
25
#include "definitions.h"
26
#include "instruction.pb.h"
27
#include "evaluator.h"
28
#include "executor.h"
29
#include "random_generator.h"
30
#include "test_util.h"
31
#include "util.h"
32
#include "gmock/gmock.h"
33
#include "gtest/gtest.h"
34
#include "absl/strings/str_cat.h"
35


36
namespace automl_zero {
37


38
using ::absl::StrCat;
39
using ::std::function;
40
using ::std::mt19937;
41
using test_only::GenerateTask;
42


43
constexpr IntegerT kNumTrainExamples = 1000;
44
constexpr IntegerT kNumValidExamples = 100;
45
constexpr double kLargeMaxAbsError = 1000000000.0;
46


47
TEST(GeneratorTest, NoOpHasNoOpInstructions) {
48
  Generator generator(
49
      NO_OP_ALGORITHM,  // Irrelevant.
50
      10,  // setup_size_init
51
      12,  // predict_size_init
52
      13,  // learn_size_init
53
      {},  // allowed_setup_ops, irrelevant.
54
      {},  // allowed_predict_ops, irrelevant.
55
      {},  // allowed_learn_ops, irrelevant.
56
      nullptr,  // bit_gen, irrelevant.
57
      nullptr);  // rand_gen, irrelevant.
58
  const InstructionIndexT setup_instruction_index = 2;
59
  const InstructionIndexT predict_instruction_index = 1;
60
  const InstructionIndexT learn_instruction_index = 3;
61
  Algorithm algorithm = generator.NoOp();
62
  EXPECT_EQ(algorithm.setup_[setup_instruction_index]->op_, NO_OP);
63
  EXPECT_EQ(algorithm.setup_[setup_instruction_index]->in1_, 0);
64
  EXPECT_EQ(algorithm.setup_[setup_instruction_index]->in2_, 0);
65
  EXPECT_EQ(algorithm.setup_[setup_instruction_index]->out_, 0);
66
  EXPECT_EQ(algorithm.setup_[setup_instruction_index]->GetActivationData(),
67
            0.0);
68
  EXPECT_EQ(algorithm.setup_[setup_instruction_index]->GetFloatData0(), 0.0);
69
  EXPECT_EQ(algorithm.setup_[setup_instruction_index]->GetFloatData1(), 0.0);
70
  EXPECT_EQ(algorithm.setup_[setup_instruction_index]->GetFloatData2(), 0.0);
71
  EXPECT_EQ(algorithm.predict_[predict_instruction_index]->op_, NO_OP);
72
  EXPECT_EQ(algorithm.predict_[predict_instruction_index]->in1_, 0);
73
  EXPECT_EQ(algorithm.predict_[predict_instruction_index]->in2_, 0);
74
  EXPECT_EQ(algorithm.predict_[predict_instruction_index]->out_, 0);
75
  EXPECT_EQ(algorithm.predict_[predict_instruction_index]->GetActivationData(),
76
            0.0);
77
  EXPECT_EQ(algorithm.predict_[predict_instruction_index]->GetFloatData0(),
78
            0.0);
79
  EXPECT_EQ(algorithm.predict_[predict_instruction_index]->GetFloatData1(),
80
            0.0);
81
  EXPECT_EQ(algorithm.predict_[predict_instruction_index]->GetFloatData2(),
82
            0.0);
83
  EXPECT_EQ(algorithm.learn_[learn_instruction_index]->op_, NO_OP);
84
  EXPECT_EQ(algorithm.learn_[learn_instruction_index]->in1_, 0);
85
  EXPECT_EQ(algorithm.learn_[learn_instruction_index]->in2_, 0);
86
  EXPECT_EQ(algorithm.learn_[learn_instruction_index]->out_, 0);
87
  EXPECT_EQ(algorithm.learn_[learn_instruction_index]->GetActivationData(),
88
            0.0);
89
  EXPECT_EQ(algorithm.learn_[learn_instruction_index]->GetFloatData0(), 0.0);
90
  EXPECT_EQ(algorithm.learn_[learn_instruction_index]->GetFloatData1(), 0.0);
91
  EXPECT_EQ(algorithm.learn_[learn_instruction_index]->GetFloatData2(), 0.0);
92
}
93


94
TEST(GeneratorTest, NoOpProducesCorrectComponentFunctionSize) {
95
  Generator generator(
96
      NO_OP_ALGORITHM,  // Irrelevant.
97
      10,  // setup_size_init
98
      12,  // predict_size_init
99
      13,  // learn_size_init
100
      {},  // allowed_setup_ops, irrelevant.
101
      {},  // allowed_predict_ops, irrelevant.
102
      {},  // allowed_learn_ops, irrelevant.
103
      nullptr,  // bit_gen, irrelevant.
104
      nullptr);  // rand_gen, irrelevant.
105
  Algorithm algorithm = generator.NoOp();
106
  EXPECT_EQ(algorithm.setup_.size(), 10);
107
  EXPECT_EQ(algorithm.predict_.size(), 12);
108
  EXPECT_EQ(algorithm.learn_.size(), 13);
109
}
110


111
TEST(GeneratorTest, Gz_Learns) {
112
  Generator generator(
113
      NO_OP_ALGORITHM,  // Irrelevant.
114
      10,  // setup_size_init, irrelevant
115
      12,  // predict_size_init, irrelevant
116
      13,  // learn_size_init, irrelevant
117
      {},  // allowed_setup_ops, irrelevant.
118
      {},  // allowed_predict_ops, irrelevant.
119
      {},  // allowed_learn_ops, irrelevant.
120
      nullptr,  // bit_gen, irrelevant.
121
      nullptr);  // rand_gen, irrelevant.
122
  Task<4> dataset =
123
      GenerateTask<4>(StrCat("scalar_linear_regression_task {} "
124
                                "num_train_examples: ",
125
                                kNumTrainExamples,
126
                                " "
127
                                "num_valid_examples: ",
128
                                kNumValidExamples,
129
                                " "
130
                                "eval_type: RMS_ERROR "
131
                                "param_seeds: 100 "
132
                                "data_seeds: 1000 "));
133
  Algorithm algorithm = generator.LinearModel(kDefaultLearningRate);
134
  mt19937 bit_gen(10000);
135
  RandomGenerator rand_gen(&bit_gen);
136
  Executor<4> executor(algorithm, dataset, kNumTrainExamples, kNumValidExamples,
137
                       &rand_gen, kLargeMaxAbsError);
138
  double fitness = executor.Execute();
139
  std::cout << "Gz_Learns fitness = " << fitness << std::endl;
140
  EXPECT_GE(fitness, 0.0);
141
  EXPECT_LE(fitness, 1.0);
142
  EXPECT_GT(fitness, 0.999);
143
}
144


145
TEST(GeneratorTest, LinearModel_Learns) {
146
  Generator generator(
147
      NO_OP_ALGORITHM,  // Irrelevant.
148
      10,  // setup_size_init, irrelevant
149
      12,  // predict_size_init, irrelevant
150
      13,  // learn_size_init, irrelevant
151
      {},  // allowed_setup_ops, irrelevant.
152
      {},  // allowed_predict_ops, irrelevant.
153
      {},  // allowed_learn_ops, irrelevant.
154
      nullptr,  // bit_gen, irrelevant.
155
      nullptr);  // rand_gen, irrelevant.
156
  Task<4> dataset =
157
      GenerateTask<4>(StrCat("scalar_linear_regression_task {} "
158
                                "num_train_examples: ",
159
                                kNumTrainExamples,
160
                                " "
161
                                "num_valid_examples: ",
162
                                kNumValidExamples,
163
                                " "
164
                                "eval_type: RMS_ERROR "
165
                                "param_seeds: 100 "
166
                                "data_seeds: 1000 "));
167
  Algorithm algorithm = generator.LinearModel(kDefaultLearningRate);
168
  mt19937 bit_gen(10000);
169
  RandomGenerator rand_gen(&bit_gen);
170
  Executor<4> executor(algorithm, dataset, kNumTrainExamples, kNumValidExamples,
171
                       &rand_gen, kLargeMaxAbsError);
172
  double fitness = executor.Execute();
173
  std::cout << "Gz_Learns fitness = " << fitness << std::endl;
174
  EXPECT_GE(fitness, 0.0);
175
  EXPECT_LE(fitness, 1.0);
176
  EXPECT_GT(fitness, 0.999);
177
}
178


179
TEST(GeneratorTest, GrTildeGrWithBias_PermanenceTest) {
180
  Generator generator(
181
      NO_OP_ALGORITHM,  // Irrelevant.
182
      0,  // setup_size_init, irrelevant.
183
      0,  // predict_size_init, irrelevant.
184
      0,  // learn_size_init, irrelevant.
185
      {},  // allowed_setup_ops, irrelevant.
186
      {},  // allowed_predict_ops, irrelevant.
187
      {},  // allowed_learn_ops, irrelevant.
188
      nullptr,  // bit_gen, irrelevant.
189
      nullptr);  // rand_gen, irrelevant.
190
  Task<4> dataset = GenerateTask<4>(StrCat(
191
      "scalar_2layer_nn_regression_task {} "
192
      "num_train_examples: ", kNumTrainExamples, " "
193
      "num_valid_examples: ", kNumValidExamples, " "
194
      "num_tasks: 1 "
195
      "eval_type: RMS_ERROR "
196
      "param_seeds: 1000 "
197
      "data_seeds: 10000 "));
198
  Algorithm algorithm = generator.NeuralNet(
199
      kDefaultLearningRate, kDefaultInitScale, kDefaultInitScale);
200
  mt19937 bit_gen(10000);
201
  RandomGenerator rand_gen(&bit_gen);
202
  Executor<4> executor(algorithm, dataset, kNumTrainExamples, kNumValidExamples,
203
                       &rand_gen, kLargeMaxAbsError);
204
  double fitness = executor.Execute();
205
  std::cout << "GrTildeGrWithBias_PermanenceTest fitness = " << fitness
206
            << std::endl;
207
  EXPECT_FLOAT_EQ(fitness, 0.80256736);
208
}
209


210
TEST(GeneratorTest, RandomInstructions) {
211
  mt19937 bit_gen;
212
  RandomGenerator rand_gen(&bit_gen);
213
  Generator generator(
214
      NO_OP_ALGORITHM,  // Irrelevant.
215
      2,           // setup_size_init
216
      4,           // predict_size_init
217
      5,           // learn_size_init
218
      {NO_OP, SCALAR_SUM_OP, MATRIX_VECTOR_PRODUCT_OP, VECTOR_MEAN_OP},
219
      {NO_OP, SCALAR_SUM_OP, MATRIX_VECTOR_PRODUCT_OP, VECTOR_MEAN_OP},
220
      {NO_OP, SCALAR_SUM_OP, MATRIX_VECTOR_PRODUCT_OP, VECTOR_MEAN_OP},
221
      &bit_gen,  // bit_gen
222
      &rand_gen);  // rand_gen
223
  const Algorithm no_op_algorithm = generator.NoOp();
224
  const IntegerT total_instructions = 2 + 4 + 5;
225
  EXPECT_TRUE(IsEventually(
226
      function<IntegerT(void)>([&](){
227
        Algorithm random_algorithm = generator.Random();
228
        return CountDifferentInstructions(random_algorithm, no_op_algorithm);
229
      }),
230
      Range<IntegerT>(0, total_instructions + 1), {total_instructions}));
231
}
232


233
TEST(GeneratorTest, RandomInstructionsProducesCorrectComponentFunctionSizes) {
234
  mt19937 bit_gen;
235
  RandomGenerator rand_gen(&bit_gen);
236
  Generator generator(
237
      NO_OP_ALGORITHM,  // Irrelevant.
238
      2,           // setup_size_init
239
      4,           // predict_size_init
240
      5,           // learn_size_init
241
      {NO_OP, SCALAR_SUM_OP, MATRIX_VECTOR_PRODUCT_OP, VECTOR_MEAN_OP},
242
      {NO_OP, SCALAR_SUM_OP, MATRIX_VECTOR_PRODUCT_OP, VECTOR_MEAN_OP},
243
      {NO_OP, SCALAR_SUM_OP, MATRIX_VECTOR_PRODUCT_OP, VECTOR_MEAN_OP},
244
      &bit_gen,  // bit_gen
245
      &rand_gen);  // rand_gen
246
  Algorithm algorithm = generator.Random();
247
  EXPECT_EQ(algorithm.setup_.size(), 2);
248
  EXPECT_EQ(algorithm.predict_.size(), 4);
249
  EXPECT_EQ(algorithm.learn_.size(), 5);
250
}
251


252
TEST(GeneratorTest, GzHasCorrectComponentFunctionSizes) {
253
  Generator generator(
254
      NO_OP_ALGORITHM,  // Irrelevant.
255
      0,  // setup_size_init, no padding.
256
      0,  // predict_size_init, no padding.
257
      0,  // learn_size_init, no padding.
258
      {},  // allowed_setup_ops, irrelevant.
259
      {},  // allowed_predict_ops, irrelevant.
260
      {},  // allowed_learn_ops, irrelevant.
261
      nullptr,  // bit_gen, irrelevant.
262
      nullptr);  // rand_gen, irrelevant.
263
  Algorithm algorithm = generator.LinearModel(kDefaultLearningRate);
264
  EXPECT_EQ(algorithm.setup_.size(), 1);
265
  EXPECT_EQ(algorithm.predict_.size(), 1);
266
  EXPECT_EQ(algorithm.learn_.size(), 4);
267
}
268


269
TEST(GeneratorTest, GzTildeGzHasCorrectComponentFunctionSizes) {
270
  Generator generator(
271
      NO_OP_ALGORITHM,  // Irrelevant.
272
      0,  // setup_size_init, no padding.
273
      0,  // predict_size_init, no padding.
274
      0,  // learn_size_init, no padding.
275
      {},  // allowed_setup_ops, irrelevant.
276
      {},  // allowed_predict_ops, irrelevant.
277
      {},  // allowed_learn_ops, irrelevant.
278
      nullptr,  // bit_gen, irrelevant.
279
      nullptr);  // rand_gen, irrelevant.
280
  Algorithm algorithm =
281
      generator.UnitTestNeuralNetNoBiasNoGradient(kDefaultLearningRate);
282
  EXPECT_EQ(algorithm.setup_.size(), 1);
283
  EXPECT_EQ(algorithm.predict_.size(), 3);
284
  EXPECT_EQ(algorithm.learn_.size(), 9);
285
}
286


287
TEST(GeneratorTest, GzTildeGzPadsComponentFunctionSizesCorrectly) {
288
  Generator generator(
289
      NO_OP_ALGORITHM,  // Irrelevant.
290
      10,  // setup_size_init
291
      12,  // predict_size_init
292
      13,  // learn_size_init
293
      {},  // allowed_setup_ops, irrelevant.
294
      {},  // allowed_predict_ops, irrelevant.
295
      {},  // allowed_learn_ops, irrelevant.
296
      nullptr,  // bit_gen, irrelevant.
297
      nullptr);  // rand_gen, irrelevant.
298
  Algorithm algorithm =
299
      generator.UnitTestNeuralNetNoBiasNoGradient(kDefaultLearningRate);
300
  EXPECT_EQ(algorithm.setup_.size(), 10);
301
  EXPECT_EQ(algorithm.predict_.size(), 12);
302
  EXPECT_EQ(algorithm.learn_.size(), 13);
303
}
304


305
TEST(GeneratorTest, GrTildeGrPadsComponentFunctionSizesCorrectly) {
306
  Generator generator(
307
      NO_OP_ALGORITHM,  // Irrelevant.
308
      16,  // setup_size_init
309
      18,  // predict_size_init
310
      19,  // learn_size_init
311
      {},  // allowed_setup_ops, irrelevant.
312
      {},  // allowed_predict_ops, irrelevant.
313
      {},  // allowed_learn_ops, irrelevant.
314
      nullptr,  // bit_gen, irrelevant.
315
      nullptr);  // rand_gen, irrelevant.
316
  Algorithm algorithm = generator.NeuralNet(
317
      kDefaultLearningRate, kDefaultInitScale, kDefaultInitScale);
318
  EXPECT_EQ(algorithm.setup_.size(), 16);
319
  EXPECT_EQ(algorithm.predict_.size(), 18);
320
  EXPECT_EQ(algorithm.learn_.size(), 19);
321
}
322


323
TEST(GeneratorTest, GzPadsComponentFunctionSizesCorrectly) {
324
  Generator generator(
325
      NO_OP_ALGORITHM,  // Irrelevant.
326
      10,  // setup_size_init
327
      12,  // predict_size_init
328
      13,  // learn_size_init
329
      {},  // allowed_setup_ops, irrelevant.
330
      {},  // allowed_predict_ops, irrelevant.
331
      {},  // allowed_learn_ops, irrelevant.
332
      nullptr,  // bit_gen, irrelevant.
333
      nullptr);  // rand_gen, irrelevant.
334
  Algorithm algorithm = generator.LinearModel(kDefaultLearningRate);
335
  EXPECT_EQ(algorithm.setup_.size(), 10);
336
  EXPECT_EQ(algorithm.predict_.size(), 12);
337
  EXPECT_EQ(algorithm.learn_.size(), 13);
338
}
339


340
}  // namespace automl_zero
341

Использование cookies

Мы используем файлы cookie в соответствии с Политикой конфиденциальности и Политикой использования cookies.

Нажимая кнопку «Принимаю», Вы даете АО «СберТех» согласие на обработку Ваших персональных данных в целях совершенствования нашего веб-сайта и Сервиса GitVerse, а также повышения удобства их использования.

Запретить использование cookies Вы можете самостоятельно в настройках Вашего браузера.