Optimizer error #233
Comments
|
Checking, it seems to be caused by multiple textencoder in sdxl. |
|
This seems to be still happening. The return type of function
This change was introduced week ago: 9e8dd33#diff-9ddf7f6762885b7ceb11cebe67daebe832b9f909e3f951ce0733ef47d4c221a5L631 I'm not sure why |
I'm using the latest SD3 branch, where kohya has added the new lr_descriptions modification |
You should not assume LyCORIS will work in latest env especially non-main branch |
|
@KohakuBlueleaf Apparently this change was introduced in dev branch and just got merged today. Sorry for taking your time! (This is PR that merged today: kohya-ss/sd-scripts#1879) |
I get this error after recent commits
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ /content/kohya-trainer/sdxl_train_network.py:174 in │
│ │
│ 171 │ args = train_util.read_config_from_file(args, parser) │
│ 172 │ │
│ 173 │ trainer = SdxlNetworkTrainer() │
│ ❱ 174 │ trainer.train(args) │
│ 175 │
│ │
│ /content/kohya-trainer/train_network.py:324 in train │
│ │
│ 321 │ │ │ ) │
│ 322 │ │ │ trainable_params = network.prepare_optimizer_params(args.text_encoder_lr, ar │
│ 323 │ │ │
│ ❱ 324 │ │ optimizer_name, optimizer_args, optimizer = train_util.get_optimizer(args, train │
│ 325 │ │ │
│ 326 │ │ # dataloaderを準備する │
│ 327 │ │ # DataLoaderのプロセス数：0はメインプロセスになる │
│ │
│ /content/kohya-trainer/library/train_util.py:3373 in get_optimizer │
│ │
│ 3370 │ │ optimizer_class = transformers.optimization.Adafactor │
│ 3371 │ │ print("OPTTTTTTTTTTTTTTT") │
│ 3372 │ │ print(trainable_params) │
│ ❱ 3373 │ │ optimizer = optimizer_class(trainable_params, lr=lr, **optimizer_kwargs) │
│ 3374 │ │
│ 3375 │ elif optimizer_type == "AdamW".lower(): │
│ 3376 │ │ print(f"use AdamW optimizer | {optimizer_kwargs}") │
│ │
│ /usr/local/lib/python3.10/dist-packages/transformers/optimization.py:607 in init │
│ │
│ 604 │ │ │ "relative_step": relative_step, │
│ 605 │ │ │ "warmup_init": warmup_init, │
│ 606 │ │ } │
│ ❱ 607 │ │ super().init(params, defaults) │
│ 608 │ │
│ 609 │ @staticmethod │
│ 610 │ def _get_lr(param_group, param_state): │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/optim/optimizer.py:371 in init │
│ │
│ 368 │ │ │ param_groups = [{"params": param_groups}] │
│ 369 │ │ │
│ 370 │ │ for param_group in param_groups: │
│ ❱ 371 │ │ │ self.add_param_group(cast(dict, param_group)) │
│ 372 │ │ │
│ 373 │ │ # Allows _cuda_graph_capture_health_check to rig a poor man's TORCH_WARN_ONCE in │
│ 374 │ │ # which I don't think exists │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/_compile.py:32 in inner │
│ │
│ 29 │ │ │ │ disable_fn = torch._dynamo.disable(fn, recursive) │
│ 30 │ │ │ │ fn.__dynamo_disable = disable_fn │
│ 31 │ │ │ │
│ ❱ 32 │ │ │ return disable_fn(*args, **kwargs) │
│ 33 │ │ │
│ 34 │ │ return inner │
│ 35 │ else: │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/_dynamo/eval_frame.py:632 in _fn │
│ │
│ 629 │ │ def _fn(*args, **kwargs): │
│ 630 │ │ │ prior = _maybe_set_eval_frame(callback) │
│ 631 │ │ │ try: │
│ ❱ 632 │ │ │ │ return fn(*args, **kwargs) │
│ 633 │ │ │ finally: │
│ 634 │ │ │ │ _maybe_set_eval_frame(prior) │
│ 635 │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/optim/optimizer.py:1019 in add_param_group │
│ │
│ 1016 │ │ │
│ 1017 │ │ for param in param_group["params"]: │
│ 1018 │ │ │ if not isinstance(param, torch.Tensor): │
│ ❱ 1019 │ │ │ │ raise TypeError( │
│ 1020 │ │ │ │ │ "optimizer can only optimize Tensors, " │
│ 1021 │ │ │ │ │ "but one of the params is " + torch.typename(param) │
│ 1022 │ │ │ │ ) │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
TypeError: optimizer can only optimize Tensors, but one of the params is list
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ /usr/local/bin/accelerate:8 in │
│ │
│ 5 from accelerate.commands.accelerate_cli import main │
│ 6 if name == 'main': │
│ 7 │ sys.argv[0] = re.sub(r'(-script.pyw|.exe)?$', '', sys.argv[0]) │
│ ❱ 8 │ sys.exit(main()) │
│ 9 │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/accelerate_cli.py:45 in main │
│ │
│ 42 │ │ exit(1) │
│ 43 │ │
│ 44 │ # Run │
│ ❱ 45 │ args.func(args) │
│ 46 │
│ 47 │
│ 48 if name == "main": │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py:918 in launch_command │
│ │
│ 915 │ elif defaults is not None and defaults.compute_environment == ComputeEnvironment.AMA │
│ 916 │ │ sagemaker_launcher(defaults, args) │
│ 917 │ else: │
│ ❱ 918 │ │ simple_launcher(args) │
│ 919 │
│ 920 │
│ 921 def main(): │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py:580 in simple_launcher │
│ │
│ 577 │ process.wait() │
│ 578 │ if process.returncode != 0: │
│ 579 │ │ if not args.quiet: │
│ ❱ 580 │ │ │ raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd) │
│ 581 │ │ else: │
│ 582 │ │ │ sys.exit(1) │
│ 583 │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
CalledProcessError: Command '['/usr/bin/python3', 'sdxl_train_network.py',
'--sample_prompts=/content/LoRA/config/sample_prompt.toml',
'--config_file=/content/LoRA/config/config_file.toml']' returned non-zero exit status 1.
The text was updated successfully, but these errors were encountered: