[dev] Easier release: cherry-pick, changelog, contributor (#11478)

Add script for  easier release including cherry pick,
generating changelog and contributor list

* Auto cherry-pick: `python release.py cherry-pick`
* Generate changelog: `python release.py changelog`
* Generate contributor: `python release.py contributor`

close: #11289
related: #12222

tools/release/README.md

@@ -0,0 +1,38 @@
+# Tools Release
+
+A tools for convenient release DolphinScheduler.
+
+## Prepare
+
+* python: python 3.6 or above
+* pip: latest version of pip is better
+
+To install dependence, you should run command
+
+```shell
+python -m pip install -r requirements.txt
+```
+
+## Usage
+
+### Export Environment Variable
+
+You can create new token in [create token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token),
+it is only need all permission under `repo`
+
+```shell
+export GH_ACCESS_TOKEN="<YOUR-GITHUB-TOKEN-WITH-REPO-ACCESS>"
+export GH_REPO_MILESTONE="<YOUR-MILESTONE>"
+```
+
+### Help
+
+```shell
+python release.py -h
+```
+
+### Action
+
+* Auto cherry-pick: `python release.py cherry-pick`, will cause error when your default branch is not up-to-date, or cherry-pick with conflict. But if you fix you can directly re-run this command, it will continue the pick
+* Generate changelog: `python release.py changelog`
+* Generate contributor: `python release.py contributor`

tools/release/github/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

tools/release/github/changelog.py

@@ -0,0 +1,151 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Github utils for release changelog."""
+
+from typing import Dict, List
+
+
+class Changelog:
+    """Generate changelog according specific pull requests list.
+
+    Each pull requests will only once in final result. If pull requests have more than one label we need,
+    will classify to high priority label type, currently priority is
+    `feature > bug > improvement > document > chore`. pr will into feature section if it with both `feature`,
+    `improvement`, `document` label.
+
+    :param prs: pull requests list.
+    """
+
+    key_number = "number"
+    key_labels = "labels"
+    key_name = "name"
+
+    label_feature = "feature"
+    label_bug = "bug"
+    label_improvement = "improvement"
+    label_document = "document"
+    label_chore = "chore"
+
+    changelog_prefix = "\n\n<details><summary>Click to expand</summary>\n\n"
+    changelog_suffix = "\n\n</details>\n"
+
+    def __init__(self, prs: List[Dict]):
+        self.prs = prs
+        self.features = []
+        self.bugfixs = []
+        self.improvements = []
+        self.documents = []
+        self.chores = []
+
+    def generate(self) -> str:
+        """Generate changelog."""
+        self.classify()
+        final = []
+        if self.features:
+            detail = f"## Feature{self.changelog_prefix}{self._convert(self.features)}{self.changelog_suffix}"
+            final.append(detail)
+        if self.improvements:
+            detail = (
+                f"## Improvement{self.changelog_prefix}"
+                f"{self._convert(self.improvements)}{self.changelog_suffix}"
+            )
+            final.append(detail)
+        if self.bugfixs:
+            detail = f"## Bugfix{self.changelog_prefix}{self._convert(self.bugfixs)}{self.changelog_suffix}"
+            final.append(detail)
+        if self.documents:
+            detail = (
+                f"## Document{self.changelog_prefix}"
+                f"{self._convert(self.documents)}{self.changelog_suffix}"
+            )
+            final.append(detail)
+        if self.chores:
+            detail = f"## Chore{self.changelog_prefix}{self._convert(self.chores)}{self.changelog_suffix}"
+            final.append(detail)
+        return "\n".join(final)
+
+    @staticmethod
+    def _convert(prs: List[Dict]) -> str:
+        """Convert pull requests into changelog item text."""
+        return "\n".join(
+            [f"- {pr['title']} (#{pr['number']}) @{pr['user']['login']}" for pr in prs]
+        )
+
+    def classify(self) -> None:
+        """Classify pull requests different kinds of section in changelog.
+
+        Each pull requests only belongs to one single classification.
+        """
+        for pr in self.prs:
+            if self.key_labels not in pr:
+                raise KeyError("PR %s do not have labels", pr[self.key_number])
+            if self._is_feature(pr):
+                self.features.append(pr)
+            elif self._is_bugfix(pr):
+                self.bugfixs.append(pr)
+            elif self._is_improvement(pr):
+                self.improvements.append(pr)
+            elif self._is_document(pr):
+                self.documents.append(pr)
+            elif self._is_chore(pr):
+                self.chores.append(pr)
+            else:
+                raise KeyError(
+                    "There must at least one of labels `feature|bug|improvement|document|chore`"
+                    "but it do not, pr: %s",
+                    pr["html_url"],
+                )
+
+    def _is_feature(self, pr: Dict) -> bool:
+        """Belong to feature pull requests."""
+        return any(
+            [
+                label[self.key_name] == self.label_feature
+                for label in pr[self.key_labels]
+            ]
+        )
+
+    def _is_bugfix(self, pr: Dict) -> bool:
+        """Belong to bugfix pull requests."""
+        return any(
+            [label[self.key_name] == self.label_bug for label in pr[self.key_labels]]
+        )
+
+    def _is_improvement(self, pr: Dict) -> bool:
+        """Belong to improvement pull requests."""
+        return any(
+            [
+                label[self.key_name] == self.label_improvement
+                for label in pr[self.key_labels]
+            ]
+        )
+
+    def _is_document(self, pr: Dict) -> bool:
+        """Belong to document pull requests."""
+        return any(
+            [
+                label[self.key_name] == self.label_document
+                for label in pr[self.key_labels]
+            ]
+        )
+
+    def _is_chore(self, pr: Dict) -> bool:
+        """Belong to chore pull requests."""
+        return any(
+            [label[self.key_name] == self.label_chore for label in pr[self.key_labels]]
+        )

tools/release/github/git.py

@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Github utils for git operations."""
+
+from pathlib import Path
+from typing import Dict, Optional
+
+from git import Repo
+
+git_dir_path: Path = Path(__file__).parent.parent.parent.parent.joinpath(".git")
+
+
+class Git:
+    """Operator to handle git object.
+
+    :param path: git repository path
+    :param branch: branch you want to query
+    """
+
+    def __init__(
+        self, path: Optional[str] = git_dir_path, branch: Optional[str] = None
+    ):
+        self.path = path
+        self.branch = branch
+
+    @property
+    def repo(self) -> Repo:
+        """Get git repo object."""
+        return Repo(self.path)
+
+    def has_commit_current(self, sha: str) -> bool:
+        """Whether SHA in current branch."""
+        branches = self.repo.git.branch("--contains", sha)
+        return f"* {self.repo.active_branch.name}" in branches
+
+    def has_commit_global(self, sha: str) -> bool:
+        """Whether SHA in all branches."""
+        try:
+            self.repo.commit(sha)
+            return True
+        except ValueError:
+            return False
+
+    def cherry_pick_pr(self, pr: Dict) -> None:
+        """Run command `git cherry-pick -x <SHA>`."""
+        sha = pr["merge_commit_sha"]
+        if not self.has_commit_global(sha):
+            raise RuntimeError(
+                "Cherry-pick SHA %s error because SHA not exists,"
+                "please make sure you local default branch is up-to-date",
+                sha,
+            )
+        if self.has_commit_current(sha):
+            print("SHA %s already in current active branch, skip it.", sha)
+        self.repo.git.cherry_pick("-x", sha)

tools/release/github/pull_request.py

@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Optional, List, Dict
+from github.resp_get import RespGet
+
+
+class PullRequest:
+    """Pull request to filter the by specific condition.
+
+    :param token: token to request GitHub API entrypoint.
+    :param repo: GitHub repository identify, use `user/repo` or `org/repo`.
+    """
+    url_search = "https://api.github.com/search/issues"
+    url_pr = "https://api.github.com/repos/{}/pulls/{}"
+
+    def __init__(self, token: str, repo: Optional[str] = "apache/dolphinscheduler"):
+        self.token = token
+        self.repo = repo
+        self.headers = {
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"token {token}",
+        }
+
+    def get_merged_detail(self, number: str) -> Dict:
+        """Get all merged pull requests detail by pr number.
+
+        :param number: pull requests number you want to get detail.
+        """
+        return RespGet(url=self.url_pr.format(self.repo, number), headers=self.headers).get_single()
+
+    def get_merged_detail_by_milestone(self, milestone: str) -> List[Dict]:
+        """Get all merged requests pull request detail by specific milestone.
+
+        :param milestone: query by specific milestone.
+        """
+        detail = []
+        numbers = {pr.get("number") for pr in self.search_merged_by_milestone(milestone)}
+        for number in numbers:
+            pr_dict = RespGet(url=self.url_pr.format(self.repo, number), headers=self.headers).get_single()
+            detail.append(pr_dict)
+        return detail
+
+    def search_merged_by_milestone(self, milestone: str) -> List[Dict]:
+        """Get all merged requests pull request by specific milestone.
+
+        :param milestone: query by specific milestone.
+        """
+        params = {"q": f"repo:{self.repo} is:pr is:merged milestone:{milestone}"}
+        return RespGet(url=self.url_search, headers=self.headers, param=params).get_total()

tools/release/github/resp_get.py

@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Github utils get HTTP response."""
+
+import copy
+import json
+from typing import Dict, List, Optional
+
+import requests
+
+
+class RespGet:
+    """Get response from GitHub restful API.
+
+    :param url: URL to requests GET method.
+    :param headers: headers for HTTP requests.
+    :param param:  param for HTTP requests.
+    """
+
+    def __init__(self, url: str, headers: dict, param: Optional[dict] = None):
+        self.url = url
+        self.headers = headers
+        self.param = param
+
+    @staticmethod
+    def get(url: str, headers: dict, params: Optional[dict] = None) -> Dict:
+        """Get single response dict from HTTP requests by given condition."""
+        resp = requests.get(url=url, headers=headers, params=params)
+        if not resp.ok:
+            raise ValueError("Requests error with", resp.reason)
+        return json.loads(resp.content)
+
+    def get_single(self) -> Dict:
+        """Get single response dict from HTTP requests by given condition."""
+        return self.get(url=self.url, headers=self.headers, params=self.param)
+
+    def get_total(self) -> List[Dict]:
+        """Get all response dict from HTTP requests by given condition.
+
+        Will change page number until no data return.
+        """
+        total = []
+        curr_param = copy.deepcopy(self.param)
+        while True:
+            curr_param["page"] = curr_param.setdefault("page", 0) + 1
+            content_dict = self.get(
+                url=self.url, headers=self.headers, params=curr_param
+            )
+            data = content_dict.get("items")
+            if not data:
+                return total
+            total.extend(data)

tools/release/github/user.py

@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Github utils for user."""
+
+from typing import Dict, List, Set
+
+
+class User:
+    """Get users according specific pull requests list.
+
+    :param prs: pull requests list.
+    """
+
+    def __init__(self, prs: List[Dict]):
+        self.prs = prs
+
+    def contribution_num(self) -> Dict:
+        """Get unique contributor with name and commit number."""
+        res = dict()
+        for pr in self.prs:
+            user_id = pr["user"]["login"]
+            res[user_id] = res.setdefault(user_id, 0) + 1
+        return res
+
+    def contributors(self) -> Set[str]:
+        """Get unique contributor with name."""
+        cn = self.contribution_num()
+        return {contributor for contributor in cn}

tools/release/release.py

@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Main function for releasing."""
+
+import argparse
+import os
+
+from github.changelog import Changelog
+from github.git import Git
+from github.pull_request import PullRequest
+from github.user import User
+
+
+def get_changelog(access_token: str, milestone: str) -> str:
+    """Get changelog in specific milestone from GitHub Restful API."""
+    pr = PullRequest(token=access_token)
+    pr_merged = pr.search_merged_by_milestone(milestone)
+    # Sort according to merged time ascending
+    pr_merged_sort = sorted(pr_merged, key=lambda p: p["closed_at"])
+
+    changelog = Changelog(pr_merged_sort)
+    changelog_text = changelog.generate()
+    return changelog_text
+
+
+def get_contributor(access_token: str, milestone: str) -> str:
+    """Get contributor in specific milestone from GitHub Restful API."""
+    pr = PullRequest(token=access_token)
+    pr_merged = pr.search_merged_by_milestone(milestone)
+
+    users = User(prs=pr_merged)
+    contributor = users.contributors()
+    # Sort according alphabetical
+    return ", ".join(sorted(contributor))
+
+
+def auto_cherry_pick(access_token: str, milestone: str) -> None:
+    """Do git cherry-pick in specific milestone, require update dev branch."""
+    pr = PullRequest(token=access_token)
+    pr_merged = pr.search_merged_by_milestone(milestone)
+    # Sort according to merged time ascending
+    pr_merged_sort = sorted(pr_merged, key=lambda p: p["closed_at"])
+
+    for p in pr_merged_sort:
+        pr_detail = pr.get_merged_detail(p["number"])
+        print(f"git cherry-pick -x {pr_detail['merge_commit_sha']}")
+        Git().cherry_pick_pr(pr_detail)
+
+
+def build_argparse() -> argparse.ArgumentParser:
+    """Build argparse.ArgumentParser with specific configuration."""
+    parser = argparse.ArgumentParser(prog="release")
+
+    subparsers = parser.add_subparsers(
+        title="subcommands",
+        dest="subcommand",
+        help="Choose one of the subcommand you want to run.",
+    )
+    parser_check = subparsers.add_parser(
+        "changelog", help="Generate changelog from specific milestone."
+    )
+    parser_check.set_defaults(func=get_changelog)
+
+    parser_prune = subparsers.add_parser(
+        "contributor", help="List all contributors from specific milestone."
+    )
+    parser_prune.set_defaults(func=get_contributor)
+
+    parser_prune = subparsers.add_parser(
+        "cherry-pick",
+        help="Auto cherry pick pr to current branch from specific milestone.",
+    )
+    parser_prune.set_defaults(func=auto_cherry_pick)
+
+    return parser
+
+
+if __name__ == "__main__":
+    arg_parser = build_argparse()
+    # args = arg_parser.parse_args(["cherry-pick"])
+    args = arg_parser.parse_args()
+
+    ENV_ACCESS_TOKEN = os.environ.get("GH_ACCESS_TOKEN", None)
+    ENV_MILESTONE = os.environ.get("GH_REPO_MILESTONE", None)
+
+    if ENV_ACCESS_TOKEN is None or ENV_MILESTONE is None:
+        raise RuntimeError(
+            "Environment variable `GH_ACCESS_TOKEN` and `GH_REPO_MILESTONE` must provider"
+        )
+
+    print(args.func(ENV_ACCESS_TOKEN, ENV_MILESTONE))

tools/release/requirements.txt

@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+requests~=2.28
+GitPython~=3.1