google-research
76 строк · 2.2 Кб
1# coding=utf-8
2# Copyright 2024 The Google Research Authors.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16"""Default Hyperparameter configuration."""
17
18import ml_collections19
20
21def get_config():22"""Gets the default hyperparameter configuration."""23
24config = ml_collections.ConfigDict()25# Exp info26config.dataset_path = "/path/to/magazine"27config.dataset = "MAGAZINE"28config.vocab_size = 13729config.experiment = "transformer"30config.model_class = "transformer"31config.image_size = 25632
33# Training info34config.seed = 035config.log_every_steps = 10036config.eval_num_steps = 100037config.max_length = 12838config.batch_size = 6439config.train_shuffle = True40config.eval_pad_last_batch = False41config.eval_batch_size = 6442config.num_train_steps = 30_00043config.checkpoint_every_steps = 500044config.eval_every_steps = 100045config.num_eval_steps = 10046
47# Model info48config.layout_dim = 249config.dtype = "float32"50config.shuffle_buffer_size = 1051config.use_vae = False52config.autoregressive = True53config.share_embeddings = True54config.num_layers = 455config.qkv_dim = 51256config.emb_dim = 51257config.mlp_dim = 204858config.num_heads = 859config.dropout_rate = 0.360config.attention_dropout_rate = 0.361config.restore_checkpoints = True62config.label_smoothing = 0.063config.sampling_method = "top-p"64config.use_vertical_info = False65
66# Optimizer info67config.optimizer = ml_collections.ConfigDict()68config.optimizer.type = "adam"69config.optimizer.warmup_steps = 400070config.optimizer.lr = 5e-471config.optimizer.beta1 = 0.972config.optimizer.beta2 = 0.9873config.optimizer.weight_decay = 0.0174config.beta_rate = 1 / 20_00075
76return config77