From b0e1b3715d5180b9ff9b41b820bbc1dbe20cbaf9 Mon Sep 17 00:00:00 2001 From: Achazwl Date: Fri, 10 Jun 2022 16:46:44 +0800 Subject: [PATCH 1/7] fix interative url --- opendelta/utils/interactive/templates/index.html | 2 +- opendelta/utils/interactive/web.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opendelta/utils/interactive/templates/index.html b/opendelta/utils/interactive/templates/index.html index bf6d411..c6235f8 100644 --- a/opendelta/utils/interactive/templates/index.html +++ b/opendelta/utils/interactive/templates/index.html @@ -165,7 +165,7 @@ for (i = 0; i < coll.length; i++) { var submit = document.getElementById("submit"); submit.addEventListener("click", function() { const Http = new XMLHttpRequest(); - const url='/submit/?name='+array.join(";"); + const url='/submit/?name='+array.join("&name="); Http.open("GET", url); Http.send(); alert("Now go back to your console") diff --git a/opendelta/utils/interactive/web.py b/opendelta/utils/interactive/web.py index a7da490..fb2c055 100644 --- a/opendelta/utils/interactive/web.py +++ b/opendelta/utils/interactive/web.py @@ -107,7 +107,7 @@ class hello: class submit: def GET(self, _): global names - names = [name.strip("root.") for name in web.input().name.split(";")] + names = [name.strip("root.") for name in web.input(name=[]).name] app.stop() def interactive(model, port=8888): From e2635545e7d1f3b1774c262a34e85a30153433cb Mon Sep 17 00:00:00 2001 From: Achazwl Date: Fri, 10 Jun 2022 17:09:45 +0800 Subject: [PATCH 2/7] fix strip --- examples/tutorial/0_interactive.py | 9 ++++++++- opendelta/utils/interactive/web.py | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/examples/tutorial/0_interactive.py b/examples/tutorial/0_interactive.py index c164a13..4bddbe4 100644 --- a/examples/tutorial/0_interactive.py +++ b/examples/tutorial/0_interactive.py @@ -2,8 +2,15 @@ from transformers import BertForMaskedLM model = BertForMaskedLM.from_pretrained("bert-base-cased") # suppose we load BERT +import sys + +if len(sys.argv) == 1: + port=True +else: + port=int(sys.argv[1]) + from opendelta import LoraModel -delta_model = LoraModel(backbone_model=model, interactive_modify=True) +delta_model = LoraModel(backbone_model=model, interactive_modify=port) # This will visualize the backbone after modification and other information. delta_model.freeze_module(exclude=["deltas", "layernorm_embedding"], set_state_dict=True) diff --git a/opendelta/utils/interactive/web.py b/opendelta/utils/interactive/web.py index fb2c055..2ff6575 100644 --- a/opendelta/utils/interactive/web.py +++ b/opendelta/utils/interactive/web.py @@ -107,7 +107,7 @@ class hello: class submit: def GET(self, _): global names - names = [name.strip("root.") for name in web.input(name=[]).name] + names = [name[5:] for name in web.input(name=[]).name] app.stop() def interactive(model, port=8888): @@ -120,7 +120,7 @@ def interactive(model, port=8888): print("If on your machine, open the link below for interactive modification.\n " "If on remote host, you could use port mapping, " "or run in vscode terminal, which automatically do port mapping for you.") - app.run() + app.run(port) global names print("modified_modules:") print(names) From c056213b973aeeb3efa0e338cec32ba80ee8e028 Mon Sep 17 00:00:00 2001 From: DingDing <32740627+ShengdingHu@users.noreply.github.com> Date: Mon, 4 Jul 2022 00:14:01 +0800 Subject: [PATCH 3/7] Update README.md --- README.md | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 06ab183..a927e87 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ OpenDelta is a toolkit for parameter efficient methods (we dub it as *delta tuni ## Updates - 2022.03.24 We notice several bugs in Soft Prompt Tuning and Prefix Tuning, mainly due to their need to customize attention ids, token_type_ids, we are fixing it! Currently, please use the other methods since they are stabler and better in performance. -- 2022.03.20 Add a [colab example](https://colab.research.google.com/drive/1hM_zu11s6plpK-YQSkz3CrowJyxtHneY?usp=sharing) to illustrate efficient training and space-saving multitask-serving. +- 2022.03.20 Add a [colab example](https://colab.research.google.com/drive/1uAhgAdc8Qr42UKYDlgUv0f7W1-gAFwGo?usp=sharing) to illustrate efficient training and space-saving multitask-serving. - 2022.03.20 A new pip version released. - 2022.02.16 Support [regular expression](https://opendelta.readthedocs.io/en/latest/notes/namebasedaddr.html#regexexpr) in named-based addressing. @@ -76,9 +76,9 @@ python setup.py develop ```python from transformers import AutoModelForSeq2SeqLM -t5 = AutoModelForSeq2SeqLM.from_pretrained("t5-base") +t5 = AutoModelForSeq2SeqLM.from_pretrained("t5-large") from opendelta import AutoDeltaModel -delta = AutoDeltaModel.from_finetuned("DeltaHub/lora_t5-base_mrpc", backbone_model=t5) +delta = AutoDeltaModel.from_finetuned("thunlp/FactQA_T5-large_Adapter", backbone_model=t5) delta.log() ``` @@ -104,13 +104,6 @@ used models that OpenDelta are sure to support. | T5-3b(parallel)| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | Deberta-v2 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | CTRL | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | -| ViT | ✅ | | | | | | | | | -## Performance Checked Combination - -Google sheet [here](https://docs.google.com/spreadsheets/d/1BIVa8ocAPga-u7rBOXLYaTfaJSjI1dWfwohmLjmFDrY/edit?usp=sharing) - -Subject to change at any moment. - From fd57f823feb0741e467eefbf2eb1b33e096144d0 Mon Sep 17 00:00:00 2001 From: William Date: Fri, 29 Jul 2022 15:48:16 +0800 Subject: [PATCH 4/7] Update structure_mapping.py --- opendelta/utils/structure_mapping.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opendelta/utils/structure_mapping.py b/opendelta/utils/structure_mapping.py index bb1f1de..4cdc507 100644 --- a/opendelta/utils/structure_mapping.py +++ b/opendelta/utils/structure_mapping.py @@ -251,7 +251,7 @@ def mapping_for_SequenceClassification(mapping, type): "out_proj": {"__name__":"out_proj"} } elif type == "bert": - mapping.pop("lm_head") + mapping.pop("cls.predictions") mapping["classifier"] = {"__name__": "classifier"} elif type == "deberta": mapping.pop("lm_predictions.lm_head") @@ -346,4 +346,4 @@ if __name__ == "__main__": for name, _ in plm.named_modules(): transform(name, t5_mapping, strict=True, warning=False) - \ No newline at end of file + From e4a0acff32d0b37455068f6afa78a1fbc64c7d42 Mon Sep 17 00:00:00 2001 From: Zhiyuan Liu Date: Tue, 23 Aug 2022 01:21:55 +0800 Subject: [PATCH 5/7] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a927e87..1f84490 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ ## Overview -OpenDelta is a toolkit for parameter efficient methods (we dub it as *delta tuning*), by which users could flexibly assign (or add) a small amount parameters to update while keeping the most paramters frozen. By using OpenDelta, users could easily implement prefix-tuning, adapters, Lora, or any other types of delta tuning with preferred PTMs. +OpenDelta is a toolkit for parameter-efficient tuning methods (we dub it as *delta tuning*), by which users could flexibly assign (or add) a small amount parameters to update while keeping the most paramters frozen. By using OpenDelta, users could easily implement prefix-tuning, adapters, Lora, or any other types of delta tuning with preferred PTMs. - Our repo is tested on Python 3.8 and PyTorch 1.9.0. Lower version may also be supported. From f6788bfc221d5b70555eb924579ce8578117937b Mon Sep 17 00:00:00 2001 From: CaffreyR <84232793+CaffreyR@users.noreply.github.com> Date: Wed, 5 Oct 2022 13:57:12 +0800 Subject: [PATCH 6/7] Update basemodel.py --- opendelta/basemodel.py | 1 + 1 file changed, 1 insertion(+) diff --git a/opendelta/basemodel.py b/opendelta/basemodel.py index 06b09c8..71f8d47 100644 --- a/opendelta/basemodel.py +++ b/opendelta/basemodel.py @@ -329,6 +329,7 @@ class DeltaBase(nn.Module, SaveLoadMixin): module (:obj:`nn.Module`, *optional*, default to :obj:`None`): The backbone model. """ + return if module is None: module = self.backbone_model device = get_device(module) From 2351259ecd7c86ecfca7084c986136dc6f22fb9d Mon Sep 17 00:00:00 2001 From: HirasawaaYui Date: Mon, 10 Oct 2022 03:23:14 +0000 Subject: [PATCH 7/7] update #31 --- opendelta/basemodel.py | 5 +++-- opendelta/delta_models/lora.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/opendelta/basemodel.py b/opendelta/basemodel.py index 71f8d47..0f6cbc6 100644 --- a/opendelta/basemodel.py +++ b/opendelta/basemodel.py @@ -92,6 +92,7 @@ class DeltaBase(nn.Module, SaveLoadMixin): default_exclude_modules = ["lm_head"] config_class = BaseDeltaConfig default_unfrozen_modules = ["deltas"] + pass_pseudo_data = True def __init__(self, backbone_model: nn.Module, modified_modules: Optional[List[str]] = None, @@ -200,7 +201,8 @@ class DeltaBase(nn.Module, SaveLoadMixin): if self.find_key(key, modified_modules): #TODO may have bugs when commonstructure has a virtual node and it's refered logger.debug("find key: {}".format(key)) self.update_module(backbone, key) - self._pseudo_data_to_instantiate(backbone) + if self.pass_pseudo_data: + self._pseudo_data_to_instantiate(backbone) # mark the paratmers that are the delta parameters for easily displaying the delta_paramters. self.mark_as_delta() return backbone @@ -329,7 +331,6 @@ class DeltaBase(nn.Module, SaveLoadMixin): module (:obj:`nn.Module`, *optional*, default to :obj:`None`): The backbone model. """ - return if module is None: module = self.backbone_model device = get_device(module) diff --git a/opendelta/delta_models/lora.py b/opendelta/delta_models/lora.py index 2c172d6..38c629a 100644 --- a/opendelta/delta_models/lora.py +++ b/opendelta/delta_models/lora.py @@ -97,6 +97,7 @@ class LoraModel(DeltaBase): config_class = LoraConfig delta_type = "lora" default_modified_modules = ['attn.q', 'attn.v'] + pass_pseudo_data = False def __init__(self, backbone_model: nn.Module, lora_r=8,