mergekit gpu 1226 #9702
mergekit gpu 1226 #9702
Conversation
|
Thanks for your contribution! |
paddlenlp/mergekit/merge_method.py
Outdated
| @@ -30,7 +31,7 @@ def merge(self, tensor_list): | |||
| elif self.merge_config.merge_type == "ties": | |||
| return self.ties(tensor_list) | |||
| else: | |||
| raise NotImplementedError(f"{self.merge_config.merge_type} is not supported yet.") | |||
| raise NotImplementedError("Unsupported tensor type.") | |||
There was a problem hiding this comment.
????为啥要修改所有的NotImplementedError("Unsupported tensor type.")
paddlenlp/mergekit/merge_method.py
Outdated
| else: | ||
| raise NotImplementedError("Paddle Tensor is not supported yet.") | ||
| raise NotImplementedError(f"Tensor type {self.merge_config.tensor_type} is not supported yet.") |
There was a problem hiding this comment.
raise NotImplementedError(f"Tensor type {self.merge_config.tensor_type} is not supported yet.") 改成 raise raise ValueError(f"Unkonwn tensor type {self.merge_config.tensor_type}")
paddlenlp/mergekit/merge_method.py
Outdated
| stacked_signs = paddle.sign(stacked_tensors).astype(mask_dtype) | ||
| majority_sign = (paddle.sum(stacked_signs, axis=0) >= 0).astype(mask_dtype) * 2 - 1 | ||
| else: | ||
| raise NotImplementedError("Unsupported tensor type.") |
| elif self.merge_config.tensor_type == "pd": | ||
| mask = paddle.cast( | ||
| paddle.bernoulli(paddle.full(tensor.shape, self.merge_config.reserve_p, dtype=tensor.dtype)), | ||
| tensor.dtype, |
There was a problem hiding this comment.
不用再cast了，输出tensor和输入tensor形状和数据类型相同https://www.paddlepaddle.org.cn/documentation/docs/zh/2.6/api/paddle/bernoulli_cn.html#bernoulli
| raise ValueError(f"Unsupported tensor type: {self.tensor_type}. Support 'np' only.") | ||
| if self.device != "cpu": | ||
| logger.warning(f"Currently only support cpu device, but got {self.device}. Setting `device` to `cpu`.") | ||
| if self.tensor_type not in ["np", "pd"]: |
There was a problem hiding this comment.
https://github.com/PaddlePaddle/PaddleNLP/blob/develop/tests/llm/test_mergekit.py 测试mergekit包括cpu numpy、cpu paddle tensor、gpu paddle tensor三种case
| tensor_list = [] | ||
|
|
||
| for i, model_path in enumerate(self.merge_config.model_path_list): | ||
| with fast_safe_open(os.path.join(model_path, index_list[i]["weight_map"][k]), framework="np") as w: |
There was a problem hiding this comment.
意识到这里的写法可能有问题，因为不同model可能存储的文件对应位置不一致
paddlenlp/mergekit/merge_model.py
Outdated
| with fast_safe_open(os.path.join(model_path, index_list[i]["weight_map"][k]), framework="np") as w: | ||
| tensor = w.get_tensor(k) | ||
| # 将 numpy 转为 Paddle Tensor | ||
| tensor = paddle.to_tensor(tensor, dtype="float32") |
There was a problem hiding this comment.
不要加dtype float32，读出来是什么dtype就用什么dtype
paddlenlp/mergekit/merge_model.py
Outdated
| framework="np", | ||
| ) as w: | ||
| base_tensor = w.get_tensor(k) | ||
| base_tensor = paddle.to_tensor(base_tensor, dtype="float32") |
| del tensor_list | ||
| if self.merge_config.base_model_path is not None: | ||
| del base_tensor | ||
|
|
There was a problem hiding this comment.
del后加上paddle.device.cuda.empty_cache()保证显存释放
| metadata={"format": "np"}, | ||
| ) | ||
| del merge_state_dict | ||
| gc.collect() |
There was a problem hiding this comment.
del后加上paddle.device.cuda.empty_cache()保证显存释放
PR types
PR changes
Description
mergekit gpu