From 44bc457396697baebe9adbae6a1304d8df4c2703 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Tue, 15 Aug 2017 13:50:49 -0400 Subject: [PATCH 01/70] Core team transition doc --- admin-docs/Core_team_transition.Rmd | 67 +++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 admin-docs/Core_team_transition.Rmd diff --git a/admin-docs/Core_team_transition.Rmd b/admin-docs/Core_team_transition.Rmd new file mode 100644 index 0000000..eb5bffe --- /dev/null +++ b/admin-docs/Core_team_transition.Rmd @@ -0,0 +1,67 @@ +--- +title: "Bioconductor Standard Repository Specification" +author: "Nitesh Turaga" +date: "8/15/2017" +output: + BiocStyle::html_document2 +--- + +## Maintain on Github + +1. All packages maintained by the Bioconductor core team will be hosted on + Github under the organization account. + +1. Maintainers within the core-team are responsible for keeping their Github and + Bioc-git repo in sync. This allows usage of many Github features, like issues, + pull requests, continuous integration and testing. + +1. Follow the scenario's to keep your package in sync, and push only to `master` + and the most recent release i.e RELEASE_3_5 (as of August 15th 2017). + +## Package Structure + +Every Bioconductor repository being maintained should look like: + +``` +git branch -a +``` + +NOTE: `master` is equivalent to SVN `devel`. Release branches, should be named + `RELEASE_X_Y`, case-sensitive. + +``` +* master + remotes/origin/HEAD -> origin/master + remotes/origin/master + remotes/origin/RELEASE_3_5 + remotes/upstream/RELEASE_2_12 + remotes/upstream/RELEASE_2_13 + remotes/upstream/RELEASE_2_14 + remotes/upstream/RELEASE_3_0 + remotes/upstream/RELEASE_3_1 + remotes/upstream/RELEASE_3_2 + remotes/upstream/RELEASE_3_3 + remotes/upstream/RELEASE_3_4 + remotes/upstream/RELEASE_3_5 + remotes/upstream/master +``` + +Every bioconductor repository should have the remotes configured properly, + +eg: BiocParallel package + +``` +git remote -v +``` + +``` +origin git@github.com:Bioconductor/BiocParallel (fetch) +origin git@github.com:Bioconductor/BiocParallel (push) +upstream git@git.bioconductor.org:packages/BiocParallel (fetch) +upstream git@git.bioconductor.org:packages/BiocParallel (push) +``` + +## Note + +- Avoid `git rebase` if you can help it. It is easier to `git fetch` and + then `git merge`. \ No newline at end of file From 53cb90b04d585eb0ca8bf3096f6cd04ca28a6f40 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Tue, 15 Aug 2017 16:59:38 -0400 Subject: [PATCH 02/70] Experiment data to add RELEASE_3_5 data This fix will add data to the RELEASE_3_5 branch of the experiment data packages. The commit before this, will undo the entire change. --- admin-docs/Core_team_transition.Rmd | 3 +-- src/git_experiment_repository.py | 27 ++++++++++++++++++++++++++- src/helper/helper.py | 20 ++++++++++++++++++++ src/run_transition.py | 8 ++++++-- 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/admin-docs/Core_team_transition.Rmd b/admin-docs/Core_team_transition.Rmd index eb5bffe..35f0130 100644 --- a/admin-docs/Core_team_transition.Rmd +++ b/admin-docs/Core_team_transition.Rmd @@ -2,8 +2,7 @@ title: "Bioconductor Standard Repository Specification" author: "Nitesh Turaga" date: "8/15/2017" -output: - BiocStyle::html_document2 +output: html_document --- ## Maintain on Github diff --git a/src/git_experiment_repository.py b/src/git_experiment_repository.py index 96bbdbe..41cc258 100644 --- a/src/git_experiment_repository.py +++ b/src/git_experiment_repository.py @@ -15,6 +15,7 @@ import subprocess from git_api.git_api import git_add from git_api.git_api import git_commit +from git_api.git_api import git_checkout import logging @@ -45,7 +46,7 @@ def list_files(self, path): for f in files] return [item[len(path) + 1:] for item in ans] - def add_data(self, package): + def add_data(self, package, release_3_5=True): """Add data from SVN data source to each package.""" package_dir = os.path.join(self.temp_git_repo, package) before_files = self.list_files(package_dir) @@ -60,6 +61,10 @@ def add_data(self, package): for ref in refs: src = (self.svn_root + self.trunk + self.data_store_path + "/" + package + "/" + ref) + if release_3_5: + src = (self.svn_root + "/" + "RELEASE_3_5" + + self.data_store_path + "/" + + package + "/" + ref) dest = "/".join([package_dir, ref]) try: cmd = ['svn', 'export', '--force', '--username', 'readonly', @@ -105,6 +110,7 @@ def run_data_transition(self, temp_git_repo): """Run data transition on all package.""" for package in os.listdir(os.path.abspath(temp_git_repo)): try: + # Skip manifest files, by checking "if" if "bioc-data-experiment" not in package: logging.info("Experiment data: Add data to package %s" % package) @@ -120,4 +126,23 @@ def run_data_transition(self, temp_git_repo): % package) logging.error(e) pass + # Checkout RELEASE_3_5 and add_data + try: + if "bioc-data-experiment" not in package: + package_dir = os.path.join(self.temp_git_repo, package) + # checkout RELEASE_3_5 in package dir + git_checkout("RELEASE_3_5", cwd=package_dir) + # Add data from branch release_3_5 + logging.info("Add data from RELEASE_3_5 %s" % package) + self.add_data(package, release_3_5=True) + logging.info("git add data to %s" % package) + self.add_data_as_git_objects(package) + logging.info("git commit data to %s" % package) + self.commit_data_as_git_objects(package) + # checkout master in package dir + git_checkout("master", cwd=package_dir) + except Exception as e: + logging.error("Experiment data: Error in add data to " + + "RELEASE_3_5 branch in " + package) + logging.error(e) return diff --git a/src/helper/helper.py b/src/helper/helper.py index a98299d..bd189e6 100644 --- a/src/helper/helper.py +++ b/src/helper/helper.py @@ -95,6 +95,26 @@ def get_union(svn_root, package_path, manifest_dictionary): return list(set(release_3_5 + release_3_6)) +def union_of_data_manifest(): + svn_root = "file:///home/git/bioc-data.hedgehog.fhcrc.org/" + release_3_5 = (svn_root + "branches/" + + "RELEASE_3_5/experiment/pkgs/" + + "bioc-data-experiment.3.5.manifest") + trunk = (svn_root + + "trunk/experiment/pkgs/" + + "bioc-data-experiment.3.6.manifest") + + def get_list(manifest): + cmd = ['svn', 'cat', manifest] + out = subprocess.check_output(cmd) + out = out.replace("## Blank lines between all entries\nPackage:", "") + package_list = [item.strip() for item in out.split("\nPackage:")] + return package_list + release_3_6 = get_list(trunk) + release_3_5 = get_list(release_3_5) + return list(set(release_3_6 + release_3_5)) + + def setup_logger(logger_name, log_file): l = logging.getLogger(logger_name) formatter = logging.Formatter('%(levelname)s : %(asctime)s : %(message)s') diff --git a/src/run_transition.py b/src/run_transition.py index ae83b61..d450ad0 100644 --- a/src/run_transition.py +++ b/src/run_transition.py @@ -17,6 +17,7 @@ from src.helper.helper import get_branch_list from src.helper.helper import get_union from src.helper.helper import populate_manifest_dictionary +from src.helper.helper import union_of_data_manifest import os import shutil import logging @@ -138,7 +139,10 @@ def run_experiment_data_transition(configfile, new_svn_dump=False): # Step 1: Initial set up, get list of packs from trunk dump = LocalSvnDump(svn_root, temp_git_repo, users_db, remote_svn_server, package_path) - packs = dump.get_pack_list(branch="trunk") + + # packs = dump.get_pack_list(branch="trunk") + # TODO: replace this hack + packs = union_of_data_manifest() ################################################### # Create a local dump of SVN packages in a location if new_svn_dump: @@ -156,7 +160,7 @@ def run_experiment_data_transition(configfile, new_svn_dump=False): lfs = Lfs(svn_root, trunk, data_store_path, ref_file, temp_git_repo) # Run make_git_repo, with new LFS object make_git_repo(svn_root, temp_git_repo, bare_git_repo, - remote_url, package_path,lfs_object=lfs) + remote_url, package_path, lfs_object=lfs) # EOF message logging.info("Completed bare git repo for experiment data packages") # FIXME: delete singleton instances From 614177145fb33901e6ef69144ff6c7cdee031dd5 Mon Sep 17 00:00:00 2001 From: nturaga Date: Wed, 16 Aug 2017 13:24:54 +0000 Subject: [PATCH 03/70] Add check to see if RELEASE_branch exists --- src/git_experiment_repository.py | 27 +++++++++++++++------------ src/run_transition.py | 1 - 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/git_experiment_repository.py b/src/git_experiment_repository.py index 41cc258..98df760 100644 --- a/src/git_experiment_repository.py +++ b/src/git_experiment_repository.py @@ -15,6 +15,7 @@ import subprocess from git_api.git_api import git_add from git_api.git_api import git_commit +from git_api.git_api import git_branch_exists from git_api.git_api import git_checkout import logging @@ -62,7 +63,8 @@ def add_data(self, package, release_3_5=True): src = (self.svn_root + self.trunk + self.data_store_path + "/" + package + "/" + ref) if release_3_5: - src = (self.svn_root + "/" + "RELEASE_3_5" + + src = (self.svn_root + "/" + "branches" + "/" + + "RELEASE_3_5" + self.data_store_path + "/" + package + "/" + ref) dest = "/".join([package_dir, ref]) @@ -130,17 +132,18 @@ def run_data_transition(self, temp_git_repo): try: if "bioc-data-experiment" not in package: package_dir = os.path.join(self.temp_git_repo, package) - # checkout RELEASE_3_5 in package dir - git_checkout("RELEASE_3_5", cwd=package_dir) - # Add data from branch release_3_5 - logging.info("Add data from RELEASE_3_5 %s" % package) - self.add_data(package, release_3_5=True) - logging.info("git add data to %s" % package) - self.add_data_as_git_objects(package) - logging.info("git commit data to %s" % package) - self.commit_data_as_git_objects(package) - # checkout master in package dir - git_checkout("master", cwd=package_dir) + if git_branch_exists("RELEASE_3_5", cwd=package_dir): + # checkout RELEASE_3_5 in package dir + git_checkout("RELEASE_3_5", cwd=package_dir) + # Add data from branch release_3_5 + logging.info("Add data from RELEASE_3_5 %s" % package) + self.add_data(package, release_3_5=True) + logging.info("git add data to %s" % package) + self.add_data_as_git_objects(package) + logging.info("git commit data to %s" % package) + self.commit_data_as_git_objects(package) + # checkout master in package dir + git_checkout("master", cwd=package_dir) except Exception as e: logging.error("Experiment data: Error in add data to " + "RELEASE_3_5 branch in " + package) diff --git a/src/run_transition.py b/src/run_transition.py index d450ad0..a4e79e8 100644 --- a/src/run_transition.py +++ b/src/run_transition.py @@ -82,7 +82,6 @@ def run_software_transition(configfile, new_svn_dump=False): # Step 1: Initial set up, get list of packs from trunk dump = LocalSvnDump(svn_root, temp_git_repo, users_db, remote_svn_server, package_path) - # packs = dump.get_pack_list(branch="trunk") manifest_dictionary = populate_manifest_dictionary(svn_root, package_path) packs = get_union(svn_root, package_path, manifest_dictionary) ################################################## From 1d8a652cbf38cb0e394eb95a9b0c98eb0f75b52c Mon Sep 17 00:00:00 2001 From: nturaga Date: Wed, 16 Aug 2017 15:29:15 +0000 Subject: [PATCH 04/70] Extremely sneaky bug, this fixes the issue of adding from RELEASE_3_5 --- src/git_experiment_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/git_experiment_repository.py b/src/git_experiment_repository.py index 98df760..702dd4d 100644 --- a/src/git_experiment_repository.py +++ b/src/git_experiment_repository.py @@ -47,7 +47,7 @@ def list_files(self, path): for f in files] return [item[len(path) + 1:] for item in ans] - def add_data(self, package, release_3_5=True): + def add_data(self, package, release_3_5=False): """Add data from SVN data source to each package.""" package_dir = os.path.join(self.temp_git_repo, package) before_files = self.list_files(package_dir) From 08776645e9bbe295b15c922ede855f5c91c78c1b Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Wed, 16 Aug 2017 11:31:32 -0400 Subject: [PATCH 05/70] Change the way lines are stripped --- src/helper/helper.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/helper/helper.py b/src/helper/helper.py index bd189e6..b9536f9 100644 --- a/src/helper/helper.py +++ b/src/helper/helper.py @@ -96,7 +96,8 @@ def get_union(svn_root, package_path, manifest_dictionary): def union_of_data_manifest(): - svn_root = "file:///home/git/bioc-data.hedgehog.fhcrc.org/" +# svn_root = "file:///home/git/bioc-data.hedgehog.fhcrc.org/" + svn_root = "https://hedgehog.fhcrc.org/bioc-data/" release_3_5 = (svn_root + "branches/" + "RELEASE_3_5/experiment/pkgs/" + "bioc-data-experiment.3.5.manifest") @@ -107,8 +108,9 @@ def union_of_data_manifest(): def get_list(manifest): cmd = ['svn', 'cat', manifest] out = subprocess.check_output(cmd) - out = out.replace("## Blank lines between all entries\nPackage:", "") - package_list = [item.strip() for item in out.split("\nPackage:")] + doc = out.split("\n") + package_list = [line.replace("Package: ","").strip() + for line in doc if line.startswith("Package")] return package_list release_3_6 = get_list(trunk) release_3_5 = get_list(release_3_5) From c90ce324c15de3f34f257a1c06e302beea43141d Mon Sep 17 00:00:00 2001 From: nturaga Date: Thu, 17 Aug 2017 15:49:24 +0000 Subject: [PATCH 06/70] Add TODO for workflow transition. Need to get a union of manifest files for running this --- src/run_transition.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/run_transition.py b/src/run_transition.py index a4e79e8..7b3d34d 100644 --- a/src/run_transition.py +++ b/src/run_transition.py @@ -282,6 +282,7 @@ def run_workflow_transition(configfile, new_svn_dump=False): ###################################### dump = LocalSvnDump(svn_root, temp_git_repo, users_db, remote_svn_server, package_path) + ## TODO: Use union of manifest files for workflow packages. packs = dump.get_pack_list(branch="trunk") # Git svn clone workflow packages if new_svn_dump: From 08eddb42b5f0e5441edda9cf66adc639dbe17d3b Mon Sep 17 00:00:00 2001 From: vobencha Date: Sun, 20 Aug 2017 07:02:25 -0700 Subject: [PATCH 07/70] git host does not allow client SSH to propagate locale information --- admin-docs/configure.md | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/admin-docs/configure.md b/admin-docs/configure.md index c256290..03b8a37 100644 --- a/admin-docs/configure.md +++ b/admin-docs/configure.md @@ -7,6 +7,8 @@ - [Smart http](#smarthttp) - [Dumb http](#dumbhttp) - [Push / Pull access via SSH](#ssh) + - [svn 'authz' to gitolite 'conf'](#sshgitolite) + - [SSH locale](#sshlocale) ## Server Specs @@ -209,8 +211,9 @@ out-of-the-box Apache configuration to limit what users can see. -- Download a package with `git clone https://git.bioconductor.org/packages/BiocGenerics.git` -## Push / pull access via ssh +## Push / pull access via SSH + ### svn 'authz' to gitolite 'conf' The gitolite configuration involves @@ -240,3 +243,38 @@ The gitolite configuration involves neaGUI.git netReg.git pairseqsim.git pgUtils.git prism.git spade.git stam.git virtualArray.git wiggleplotr.git xcmsGUI.git xmapcore.git + +### SSH locale + +See this issue for details: + +https://github.com/Bioconductor/bioc_git_transition/issues/34 + +When a user ran 'git pull' with a non-C and non-US locale, the remote +server (i.e., git.bioconductor.org) issued a perl warning: + +perl: warning: Setting locale failed. +perl: warning: Please check that your locale settings: + LANGUAGE = (unset), + LC_ALL = (unset), + LC_TIME = "en_GB.UTF-8", + LC_MONETARY = "en_GB.UTF-8", + LC_MEASUREMENT = "en_GB.UTF-8", + LC_NUMERIC = "en_GB.UTF-8", + LC_PAPER = "en_GB.UTF-8", + LANG = "en_US.UTF-8" + are supported and installed on your system. +perl: warning: Falling back to a fallback locale ("en_US.UTF-8"). + +To prevent this, the git server was modified to prevent clients +from propagating their locale variables via SSH. + +There are 2 SSH config files, one is for clients connecting to the host and the +other is for the ssh daemon running on the host. Modify the config file for +the daemon, /etc/ssh/sshd_confg, by commenting out this line + + AcceptEnv LANG LC_* + +then restart the service + + sudo service sshd restart From 725a05d52bbf725f199fa72457465bc6021cafd7 Mon Sep 17 00:00:00 2001 From: vobencha Date: Sun, 20 Aug 2017 07:05:05 -0700 Subject: [PATCH 08/70] fix format --- admin-docs/configure.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/admin-docs/configure.md b/admin-docs/configure.md index 03b8a37..1e0ed64 100644 --- a/admin-docs/configure.md +++ b/admin-docs/configure.md @@ -253,18 +253,18 @@ https://github.com/Bioconductor/bioc_git_transition/issues/34 When a user ran 'git pull' with a non-C and non-US locale, the remote server (i.e., git.bioconductor.org) issued a perl warning: -perl: warning: Setting locale failed. -perl: warning: Please check that your locale settings: - LANGUAGE = (unset), - LC_ALL = (unset), - LC_TIME = "en_GB.UTF-8", - LC_MONETARY = "en_GB.UTF-8", - LC_MEASUREMENT = "en_GB.UTF-8", - LC_NUMERIC = "en_GB.UTF-8", - LC_PAPER = "en_GB.UTF-8", - LANG = "en_US.UTF-8" - are supported and installed on your system. -perl: warning: Falling back to a fallback locale ("en_US.UTF-8"). + perl: warning: Setting locale failed. + perl: warning: Please check that your locale settings: + LANGUAGE = (unset), + LC_ALL = (unset), + LC_TIME = "en_GB.UTF-8", + LC_MONETARY = "en_GB.UTF-8", + LC_MEASUREMENT = "en_GB.UTF-8", + LC_NUMERIC = "en_GB.UTF-8", + LC_PAPER = "en_GB.UTF-8", + LANG = "en_US.UTF-8" + are supported and installed on your system. + perl: warning: Falling back to a fallback locale ("en_US.UTF-8"). To prevent this, the git server was modified to prevent clients from propagating their locale variables via SSH. From 2919f75e75dc43cc346199af69a858a2ac44603a Mon Sep 17 00:00:00 2001 From: vobencha Date: Sun, 20 Aug 2017 07:12:13 -0700 Subject: [PATCH 09/70] fix wording --- admin-docs/configure.md | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/admin-docs/configure.md b/admin-docs/configure.md index 1e0ed64..c905887 100644 --- a/admin-docs/configure.md +++ b/admin-docs/configure.md @@ -246,32 +246,22 @@ The gitolite configuration involves ### SSH locale -See this issue for details: +When a user ran 'git pull' with a non-C and non-US locale, the remote +server (i.e., git.bioconductor.org) issued a perl warning that +setting the locale failed and the fallback locale ("en_US.UTF-8") would +be used. See this issue for full details: https://github.com/Bioconductor/bioc_git_transition/issues/34 -When a user ran 'git pull' with a non-C and non-US locale, the remote -server (i.e., git.bioconductor.org) issued a perl warning: - - perl: warning: Setting locale failed. - perl: warning: Please check that your locale settings: - LANGUAGE = (unset), - LC_ALL = (unset), - LC_TIME = "en_GB.UTF-8", - LC_MONETARY = "en_GB.UTF-8", - LC_MEASUREMENT = "en_GB.UTF-8", - LC_NUMERIC = "en_GB.UTF-8", - LC_PAPER = "en_GB.UTF-8", - LANG = "en_US.UTF-8" - are supported and installed on your system. - perl: warning: Falling back to a fallback locale ("en_US.UTF-8"). - To prevent this, the git server was modified to prevent clients from propagating their locale variables via SSH. -There are 2 SSH config files, one is for clients connecting to the host and the -other is for the ssh daemon running on the host. Modify the config file for -the daemon, /etc/ssh/sshd_confg, by commenting out this line +There are 2 SSH config files, one is for clients connecting to the host +(/etc/ssh/ssh_config) and another for the ssh daemon running on the host +(/etc/ssh/sshd_config). + +Modify the config file for the daemon, /etc/ssh/sshd_confg, by commenting out +this line AcceptEnv LANG LC_* From 381a04591d65ae727e7d2e695a7c6d18e5bb5abf Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Wed, 23 Aug 2017 17:43:53 -0400 Subject: [PATCH 10/70] pre-receive hook to prevent large files and duplicate commits --- .../prevent-large-files-and-duplicate-commits | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 hooks/repo-specific/prevent-large-files-and-duplicate-commits diff --git a/hooks/repo-specific/prevent-large-files-and-duplicate-commits b/hooks/repo-specific/prevent-large-files-and-duplicate-commits new file mode 100644 index 0000000..c5dfac0 --- /dev/null +++ b/hooks/repo-specific/prevent-large-files-and-duplicate-commits @@ -0,0 +1,141 @@ +#!/usr/bin/env python + +import subprocess +import sys +import fileinput + +# +# Pre-receive hook that will block any new commits +# that contain files larger than 5Mb in size and duplicate commits +# + + +ZERO_COMMIT = "0000000000000000000000000000000000000000" +MAXSIZE = "5000000" # 5MB limit on file size +ERROR_DUPLICATE_COMMITS = """ERROR + + There are duplicate commits in your commit history, These cannot + be pushed to the Bioconductor git server. Please make sure that + this is resolved. + + Take a look at the documentation to fix this, + https://bioconductor.org/developers/how-to/git/sync-existing-repositories/, + particularly, point #8 (force Bioconductor master to Github + master). + + For more information, or help resolving this issue, contact + . Provide the error, the package name + and any other details we might need. + +Duplicate commits: + + %s + %s + +Use + + git show %s + git show %s + +to see body of commits. +""" + +ERROR_LARGE_FILE = """ERROR +%s larger than 5Mb. Please see Biocondcutor package guidelines +https://bioconductor.org/developers/package-guidelines/. +""" + + +def disable_large_files(oldrev, newrev, refname): + """Pre-receive hook to check for large files.""" + # Check for zero commit, check branch deletions + if newrev == ZERO_COMMIT: + pass + + # set oldrec properly if this is branch creation + if oldrev == ZERO_COMMIT: + oldrev = "HEAD" + + list_files = subprocess.check_out(["git", "diff", + "--name-only", "--diff-filter=ACMRT", + oldrev + ".." + newrev]) + for fl in list_files: + size = subprocess.check_out(["git", "cat-file", "-s", + newrev + ":" + fl]) + # Check to see if for some reason we didn't get a size + if not size: + # Compare filesize to MAXSIZE + if size >= MAXSIZE: + print(ERROR_LARGE_FILE % fl) + sys.exit(1) + sys.exit(0) + return + + +def check_commit_pairs(oldrev, newrev, refname): + """Pre-receive hook to check for duplicate commits.""" + if newrev == ZERO_COMMIT: + continue + try: + commit_list = subprocess.check_output(["git", + "rev-list", + newrev, "-n", "10"]) + except Exception as e: + print("Exception: %s" % e) + pass + commit_list = commit_list.split("\n") + commit_list = [item for item in commit_list if len(item)>0] + # For each of the first 10 pairs, check diff + + for i in xrange(len(commit_list) - 1): + first = commit_list[i] + second = commit_list[i+1] + + # use 'show' to test for empty commits, + # else git diff will report no diffs + body1 = subprocess.check_output(["git", "show", + "--format=%b", first]).strip() + if not body1: + continue + + body2 = subprocess.check_output(["git", "show", + "--format=%b", second]).strip() + if not body2: + continue + + # Get diff of two commits + diff = subprocess.check_output(["git", "diff", first, second]) + # If the diff of two commits is empty, means they are the same. + # i.e duplicate + if not diff: + print(ERROR_DUPLICATE_COMMITS % (first, second, first, second)) + sys.exit(1) + return + + +# This code doesn't run; enable for local testing. +# to use: +# +# 1. Toggle 'False' to 'True' +# 2. make a git repostiory +# 3. create invalid commit history +# 4. run python prevent-duplicate-commits from repository root +# 5. create branch and repeat for each test +if False: + newrev = subprocess.check_output([ + "git", "log", "-1", "--format=%H" + ]).strip() + + commit_list = subprocess.check_output([ + "git", "rev-list", "HEAD", "-n", "10" + ]) + check_commit_pairs(commit_list) + sys.exit(0) + +if __name__ == "__main__": + for line in fileinput.input(): + std_input = line.split(" ") + oldrev, newrev, refname = std_input[0], std_input[1], std_input[2] + + disable_large_files(oldrev, newrev, refname) + check_commit_pairs(oldrev, newrev, refname) From 9ede97a097664c076be58fabd5ef70208008443d Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Thu, 24 Aug 2017 11:23:16 -0400 Subject: [PATCH 11/70] Minor bug fixes The check for zero-commit needs to be in a different location in the loop --- .gitignore | 3 ++ .../prevent-large-files-and-duplicate-commits | 32 +++++++++---------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 030dc25..d6cf5ce 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ update*.svn extdata admin-docs/*Bioc2017* .gitignore +admin-docs/presentations/.ipynb_checkpoints/ +admin-docs/presentations/Demo-bioc2017.ipynb +admin-docs/presentations/core-team-transition.ipynb diff --git a/hooks/repo-specific/prevent-large-files-and-duplicate-commits b/hooks/repo-specific/prevent-large-files-and-duplicate-commits index c5dfac0..b528eb6 100644 --- a/hooks/repo-specific/prevent-large-files-and-duplicate-commits +++ b/hooks/repo-specific/prevent-large-files-and-duplicate-commits @@ -4,11 +4,7 @@ import subprocess import sys import fileinput -# -# Pre-receive hook that will block any new commits -# that contain files larger than 5Mb in size and duplicate commits -# - +# Global variables used by pre-recieve hook ZERO_COMMIT = "0000000000000000000000000000000000000000" MAXSIZE = "5000000" # 5MB limit on file size @@ -46,11 +42,13 @@ https://bioconductor.org/developers/package-guidelines/. """ -def disable_large_files(oldrev, newrev, refname): +# +# Pre-receive hook that will block any new commits +# that contain files larger than 5Mb in size and duplicate commits +# + +def prevent_large_files(oldrev, newrev, refname): """Pre-receive hook to check for large files.""" - # Check for zero commit, check branch deletions - if newrev == ZERO_COMMIT: - pass # set oldrec properly if this is branch creation if oldrev == ZERO_COMMIT: @@ -72,10 +70,8 @@ def disable_large_files(oldrev, newrev, refname): return -def check_commit_pairs(oldrev, newrev, refname): +def prevent_duplicate_commits(oldrev, newrev, refname): """Pre-receive hook to check for duplicate commits.""" - if newrev == ZERO_COMMIT: - continue try: commit_list = subprocess.check_output(["git", "rev-list", @@ -129,13 +125,17 @@ if False: commit_list = subprocess.check_output([ "git", "rev-list", "HEAD", "-n", "10" ]) - check_commit_pairs(commit_list) + prevent_duplicate_commits(commit_list) sys.exit(0) if __name__ == "__main__": for line in fileinput.input(): std_input = line.split(" ") oldrev, newrev, refname = std_input[0], std_input[1], std_input[2] - - disable_large_files(oldrev, newrev, refname) - check_commit_pairs(oldrev, newrev, refname) + # Check for zero commit, check branch deletions + if newrev == ZERO_COMMIT: + continue + # prevent large files from being committed + prevent_large_files(oldrev, newrev, refname) + # prevent duplicate commits + prevent_duplicate_commits(oldrev, newrev, refname) From 02730b374da8d1791e5668dd122e6344650fbb66 Mon Sep 17 00:00:00 2001 From: vobencha Date: Fri, 25 Aug 2017 14:48:33 -0700 Subject: [PATCH 12/70] fix formatting --- admin-docs/configure.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/admin-docs/configure.md b/admin-docs/configure.md index c905887..86f93d6 100644 --- a/admin-docs/configure.md +++ b/admin-docs/configure.md @@ -202,13 +202,13 @@ out-of-the-box Apache configuration to limit what users can see. user: read, write, execute group: read, execute other: none -- All files under /home/git/repositories should have the following permissions: - user: read, write - group: read - other: none -- Testing: - -- Paste https://git.bioconductor.org/packages/ in a browser and confirm all packages are visible. - -- Download a package with `git clone https://git.bioconductor.org/packages/BiocGenerics.git` +- All files under /home/git/repositories should have the following permissions: + user: read, write + group: read + other: none +- Testing: + -- Paste https://git.bioconductor.org/packages/ in a browser and confirm all packages are visible. + -- Download a package with `git clone https://git.bioconductor.org/packages/BiocGenerics.git` ## Push / pull access via SSH From 26618b3672bd06fc5330b66b2b1ae84cf3fca3cc Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Tue, 29 Aug 2017 09:52:54 -0400 Subject: [PATCH 13/70] Python subprocess call without writing the disable-large-file script --- ...t-large-files-and-duplicate-commits-redeux | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 hooks/repo-specific/prevent-large-files-and-duplicate-commits-redeux diff --git a/hooks/repo-specific/prevent-large-files-and-duplicate-commits-redeux b/hooks/repo-specific/prevent-large-files-and-duplicate-commits-redeux new file mode 100644 index 0000000..d63b5ec --- /dev/null +++ b/hooks/repo-specific/prevent-large-files-and-duplicate-commits-redeux @@ -0,0 +1,107 @@ +#!/usr/bin/env python + +import subprocess +import sys +import fileinput + +# Global variables used by pre-recieve hook + +ZERO_COMMIT = "0000000000000000000000000000000000000000" +ERROR_DUPLICATE_COMMITS = """ERROR + + There are duplicate commits in your commit history, These cannot + be pushed to the Bioconductor git server. Please make sure that + this is resolved. + + Take a look at the documentation to fix this, + https://bioconductor.org/developers/how-to/git/sync-existing-repositories/, + particularly, point #8 (force Bioconductor master to Github + master). + + For more information, or help resolving this issue, contact + . Provide the error, the package name + and any other details we might need. + +Duplicate commits: + + %s + %s + +Use + + git show %s + git show %s + +to see body of commits. +""" + +def prevent_large_files(): + suprocess.call(["./disable-large-commits"]) + return + +def prevent_duplicate_commits(oldrev, newrev, refname): + """Pre-receive hook to check for duplicate commits.""" + try: + commit_list = subprocess.check_output(["git", + "rev-list", + newrev, "-n", "10"]) + except Exception as e: + print("Exception: %s" % e) + pass + commit_list = commit_list.split("\n") + commit_list = [item for item in commit_list if len(item)>0] + # For each of the first 10 pairs, check diff + + for i in xrange(len(commit_list) - 1): + first = commit_list[i] + second = commit_list[i+1] + + # use 'show' to test for empty commits, + # else git diff will report no diffs + body1 = subprocess.check_output(["git", "show", + "--format=%b", first]).strip() + if not body1: + continue + + body2 = subprocess.check_output(["git", "show", + "--format=%b", second]).strip() + if not body2: + continue + + # Get diff of two commits + diff = subprocess.check_output(["git", "diff", first, second]) + # If the diff of two commits is empty, means they are the same. + # i.e duplicate + if not diff: + print(ERROR_DUPLICATE_COMMITS % (first, second, first, second)) + sys.exit(1) + return + +# This code doesn't run; enable for local testing. +# to use: +# +# 1. Toggle 'False' to 'True' +# 2. make a git repostiory +# 3. create invalid commit history +# 4. run python prevent-duplicate-commits from repository root +# 5. create branch and repeat for each test +if False: + newrev = subprocess.check_output([ + "git", "log", "-1", "--format=%H" + ]).strip() + + commit_list = subprocess.check_output([ + "git", "rev-list", "HEAD", "-n", "10" + ]) + prevent_duplicate_commits(commit_list) + sys.exit(0) + +if __name__ == "__main__": + for line in fileinput.input(): + std_input = line.split(" ") + oldrev, newrev, refname = std_input[0], std_input[1], std_input[2] + # Check for zero commit, check branch deletions + if newrev == ZERO_COMMIT: + continue + # prevent duplicate commits + prevent_duplicate_commits(oldrev, newrev, refname) From dda9cce92aaa6b54a9b32457380b7fbcf84a9377 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Tue, 29 Aug 2017 11:44:34 -0400 Subject: [PATCH 14/70] New set of pre-receive hooks for gitolite Prevent: 1. large files 2. duplicate commits --- hooks/repo-specific/pre-receive-hook | 41 +++++ .../prevent-large-files-and-duplicate-commits | 141 ------------------ ...t-large-files-and-duplicate-commits-redeux | 107 ------------- .../prevent_duplicate_commits.py | 68 +++++++++ hooks/repo-specific/prevent_large_files.py | 40 +++++ 5 files changed, 149 insertions(+), 248 deletions(-) create mode 100644 hooks/repo-specific/pre-receive-hook delete mode 100644 hooks/repo-specific/prevent-large-files-and-duplicate-commits delete mode 100644 hooks/repo-specific/prevent-large-files-and-duplicate-commits-redeux create mode 100644 hooks/repo-specific/prevent_duplicate_commits.py create mode 100644 hooks/repo-specific/prevent_large_files.py diff --git a/hooks/repo-specific/pre-receive-hook b/hooks/repo-specific/pre-receive-hook new file mode 100644 index 0000000..fd30d26 --- /dev/null +++ b/hooks/repo-specific/pre-receive-hook @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +import subprocess +import sys +import fileinput +from prevent_large_files import prevent_large_files +from prevent_duplicate_commits import prevent_duplicate_commits + + +ZERO_COMMIT = "0000000000000000000000000000000000000000" + +# This code doesn't run; enable for local testing. +# to use: +# +# 1. Toggle 'False' to 'True' +# 2. make a git repostiory +# 3. create invalid commit history +# 4. run python prevent-duplicate-commits from repository root +# 5. create branch and repeat for each test +if False: + refname=None + revs = subprocess.check_output([ + "git", "log", "-2", "--format=%H" + ]).splitlines() + newrev = revs[0].strip() + oldrev = revs[1].strip() + prevent_large_files(oldrev, newrev, refname) + prevent_duplicate_commits(oldrev, newrev, refname) + sys.exit(0) + +if __name__ == "__main__": + for line in fileinput.input(): + std_input = line.split(" ") + oldrev, newrev, refname = std_input[0], std_input[1], std_input[2] + # Check for zero commit, check branch deletions + if newrev == ZERO_COMMIT: + continue + # prevent large files + prevent_large_files(oldrev, newrev, refname) + # prevent duplicate commits + prevent_duplicate_commits(oldrev, newrev, refname) diff --git a/hooks/repo-specific/prevent-large-files-and-duplicate-commits b/hooks/repo-specific/prevent-large-files-and-duplicate-commits deleted file mode 100644 index b528eb6..0000000 --- a/hooks/repo-specific/prevent-large-files-and-duplicate-commits +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env python - -import subprocess -import sys -import fileinput - -# Global variables used by pre-recieve hook - -ZERO_COMMIT = "0000000000000000000000000000000000000000" -MAXSIZE = "5000000" # 5MB limit on file size -ERROR_DUPLICATE_COMMITS = """ERROR - - There are duplicate commits in your commit history, These cannot - be pushed to the Bioconductor git server. Please make sure that - this is resolved. - - Take a look at the documentation to fix this, - https://bioconductor.org/developers/how-to/git/sync-existing-repositories/, - particularly, point #8 (force Bioconductor master to Github - master). - - For more information, or help resolving this issue, contact - . Provide the error, the package name - and any other details we might need. - -Duplicate commits: - - %s - %s - -Use - - git show %s - git show %s - -to see body of commits. -""" - -ERROR_LARGE_FILE = """ERROR -%s larger than 5Mb. Please see Biocondcutor package guidelines -https://bioconductor.org/developers/package-guidelines/. -""" - - -# -# Pre-receive hook that will block any new commits -# that contain files larger than 5Mb in size and duplicate commits -# - -def prevent_large_files(oldrev, newrev, refname): - """Pre-receive hook to check for large files.""" - - # set oldrec properly if this is branch creation - if oldrev == ZERO_COMMIT: - oldrev = "HEAD" - - list_files = subprocess.check_out(["git", "diff", - "--name-only", "--diff-filter=ACMRT", - oldrev + ".." + newrev]) - for fl in list_files: - size = subprocess.check_out(["git", "cat-file", "-s", - newrev + ":" + fl]) - # Check to see if for some reason we didn't get a size - if not size: - # Compare filesize to MAXSIZE - if size >= MAXSIZE: - print(ERROR_LARGE_FILE % fl) - sys.exit(1) - sys.exit(0) - return - - -def prevent_duplicate_commits(oldrev, newrev, refname): - """Pre-receive hook to check for duplicate commits.""" - try: - commit_list = subprocess.check_output(["git", - "rev-list", - newrev, "-n", "10"]) - except Exception as e: - print("Exception: %s" % e) - pass - commit_list = commit_list.split("\n") - commit_list = [item for item in commit_list if len(item)>0] - # For each of the first 10 pairs, check diff - - for i in xrange(len(commit_list) - 1): - first = commit_list[i] - second = commit_list[i+1] - - # use 'show' to test for empty commits, - # else git diff will report no diffs - body1 = subprocess.check_output(["git", "show", - "--format=%b", first]).strip() - if not body1: - continue - - body2 = subprocess.check_output(["git", "show", - "--format=%b", second]).strip() - if not body2: - continue - - # Get diff of two commits - diff = subprocess.check_output(["git", "diff", first, second]) - # If the diff of two commits is empty, means they are the same. - # i.e duplicate - if not diff: - print(ERROR_DUPLICATE_COMMITS % (first, second, first, second)) - sys.exit(1) - return - - -# This code doesn't run; enable for local testing. -# to use: -# -# 1. Toggle 'False' to 'True' -# 2. make a git repostiory -# 3. create invalid commit history -# 4. run python prevent-duplicate-commits from repository root -# 5. create branch and repeat for each test -if False: - newrev = subprocess.check_output([ - "git", "log", "-1", "--format=%H" - ]).strip() - - commit_list = subprocess.check_output([ - "git", "rev-list", "HEAD", "-n", "10" - ]) - prevent_duplicate_commits(commit_list) - sys.exit(0) - -if __name__ == "__main__": - for line in fileinput.input(): - std_input = line.split(" ") - oldrev, newrev, refname = std_input[0], std_input[1], std_input[2] - # Check for zero commit, check branch deletions - if newrev == ZERO_COMMIT: - continue - # prevent large files from being committed - prevent_large_files(oldrev, newrev, refname) - # prevent duplicate commits - prevent_duplicate_commits(oldrev, newrev, refname) diff --git a/hooks/repo-specific/prevent-large-files-and-duplicate-commits-redeux b/hooks/repo-specific/prevent-large-files-and-duplicate-commits-redeux deleted file mode 100644 index d63b5ec..0000000 --- a/hooks/repo-specific/prevent-large-files-and-duplicate-commits-redeux +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python - -import subprocess -import sys -import fileinput - -# Global variables used by pre-recieve hook - -ZERO_COMMIT = "0000000000000000000000000000000000000000" -ERROR_DUPLICATE_COMMITS = """ERROR - - There are duplicate commits in your commit history, These cannot - be pushed to the Bioconductor git server. Please make sure that - this is resolved. - - Take a look at the documentation to fix this, - https://bioconductor.org/developers/how-to/git/sync-existing-repositories/, - particularly, point #8 (force Bioconductor master to Github - master). - - For more information, or help resolving this issue, contact - . Provide the error, the package name - and any other details we might need. - -Duplicate commits: - - %s - %s - -Use - - git show %s - git show %s - -to see body of commits. -""" - -def prevent_large_files(): - suprocess.call(["./disable-large-commits"]) - return - -def prevent_duplicate_commits(oldrev, newrev, refname): - """Pre-receive hook to check for duplicate commits.""" - try: - commit_list = subprocess.check_output(["git", - "rev-list", - newrev, "-n", "10"]) - except Exception as e: - print("Exception: %s" % e) - pass - commit_list = commit_list.split("\n") - commit_list = [item for item in commit_list if len(item)>0] - # For each of the first 10 pairs, check diff - - for i in xrange(len(commit_list) - 1): - first = commit_list[i] - second = commit_list[i+1] - - # use 'show' to test for empty commits, - # else git diff will report no diffs - body1 = subprocess.check_output(["git", "show", - "--format=%b", first]).strip() - if not body1: - continue - - body2 = subprocess.check_output(["git", "show", - "--format=%b", second]).strip() - if not body2: - continue - - # Get diff of two commits - diff = subprocess.check_output(["git", "diff", first, second]) - # If the diff of two commits is empty, means they are the same. - # i.e duplicate - if not diff: - print(ERROR_DUPLICATE_COMMITS % (first, second, first, second)) - sys.exit(1) - return - -# This code doesn't run; enable for local testing. -# to use: -# -# 1. Toggle 'False' to 'True' -# 2. make a git repostiory -# 3. create invalid commit history -# 4. run python prevent-duplicate-commits from repository root -# 5. create branch and repeat for each test -if False: - newrev = subprocess.check_output([ - "git", "log", "-1", "--format=%H" - ]).strip() - - commit_list = subprocess.check_output([ - "git", "rev-list", "HEAD", "-n", "10" - ]) - prevent_duplicate_commits(commit_list) - sys.exit(0) - -if __name__ == "__main__": - for line in fileinput.input(): - std_input = line.split(" ") - oldrev, newrev, refname = std_input[0], std_input[1], std_input[2] - # Check for zero commit, check branch deletions - if newrev == ZERO_COMMIT: - continue - # prevent duplicate commits - prevent_duplicate_commits(oldrev, newrev, refname) diff --git a/hooks/repo-specific/prevent_duplicate_commits.py b/hooks/repo-specific/prevent_duplicate_commits.py new file mode 100644 index 0000000..deb84a6 --- /dev/null +++ b/hooks/repo-specific/prevent_duplicate_commits.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python + +import subprocess +import sys +import fileinput + +# Global variables used by pre-recieve hook + +ZERO_COMMIT = "0000000000000000000000000000000000000000" +ERROR_DUPLICATE_COMMITS = """Error: duplicate commits. + +There are duplicate commits in your commit history, These cannot be +pushed to the Bioconductor git server. Please make sure that this is +resolved. + +Take a look at the documentation to fix this, +https://bioconductor.org/developers/how-to/git/sync-existing-repositories/, +particularly, point #8 (force Bioconductor master to Github master). + +For more information, or help resolving this issue, contact +. Provide the error, the package name and +any other details we might need. + +Use + + git show %s + git show %s + +to see body of commits. +""" + +def prevent_duplicate_commits(oldrev, newrev, refname): + """Pre-receive hook to check for duplicate commits.""" + try: + commit_list = subprocess.check_output(["git", + "rev-list", + newrev, "-n", "20"]) + except Exception as e: + print("Exception: %s" % e) + pass + commit_list = commit_list.split("\n") + commit_list = [item for item in commit_list if len(item)>0] + # For each of the first 10 pairs, check diff + + for i in xrange(len(commit_list) - 1): + first = commit_list[i] + second = commit_list[i+1] + + # use 'show' to test for empty commits, + # else git diff will report no diffs + body1 = subprocess.check_output(["git", "show", + "--format=%b", first]).strip() + if not body1: + continue + + body2 = subprocess.check_output(["git", "show", + "--format=%b", second]).strip() + if not body2: + continue + + # Get diff of two commits + diff = subprocess.check_output(["git", "diff", first, second]) + # If the diff of two commits is empty, means they are the same. + # i.e duplicate + if not diff: + print(ERROR_DUPLICATE_COMMITS % (first, second)) + sys.exit(1) + return diff --git a/hooks/repo-specific/prevent_large_files.py b/hooks/repo-specific/prevent_large_files.py new file mode 100644 index 0000000..5c17cd6 --- /dev/null +++ b/hooks/repo-specific/prevent_large_files.py @@ -0,0 +1,40 @@ +import subprocess +import sys +import fileinput +import math +# Global variables used by pre-recieve hook + +ZERO_COMMIT = "0000000000000000000000000000000000000000" +MAXSIZE = int(5000000) # 5MB limit on file size +ERROR_MSG = """Error: file larger than %.0f Mb. + + File name: '%s' + File size: %.1f Mb + +Please see Biocondcutor guidelines +https://bioconductor.org/developers/package-guidelines/ +""" + +def prevent_large_files(oldrev, newrev, refname): + """Pre-receive hook to check for large files.""" + + # set oldrec properly if this is branch creation + if oldrev == ZERO_COMMIT: + oldrev = "HEAD" + + list_files = subprocess.check_output(["git", "diff", + "--name-only", "--diff-filter=ACMRT", + oldrev + ".." + newrev]) + for fl in list_files.splitlines(): + + size = subprocess.check_output(["git", "cat-file", "-s", + newrev + ":" + fl]) + # Check to see if for some reason we didn't get a size + size = int(size.strip()) + if size: + # Compare filesize to MAXSIZE + mb = 1024.0 * 1024.0 + if size > MAXSIZE: + print(ERROR_MSG % (MAXSIZE / mb, fl, size / mb) ) + sys.exit(1) + return From fbdc00f84f255f84c0f09f1720224e078bc59e10 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Tue, 29 Aug 2017 11:45:20 -0400 Subject: [PATCH 15/70] Remove old pre recive hook --- hooks/repo-specific/disable-large-commits | 42 ----------------------- 1 file changed, 42 deletions(-) delete mode 100755 hooks/repo-specific/disable-large-commits diff --git a/hooks/repo-specific/disable-large-commits b/hooks/repo-specific/disable-large-commits deleted file mode 100755 index d752dad..0000000 --- a/hooks/repo-specific/disable-large-commits +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -# -# Pre-receive hook that will block any new commits that contain files larger than 5Mb in size. -# - -GITCMD="/usr/bin/git" -zero_commit="0000000000000000000000000000000000000000" -MAXSIZE="5000000" # 5MB limit on file size - -# Read stdin for ref information -while read oldrev newrev refname; do - # Skip branch deletions - if [ "${newrev}" = "${zero_commit}" ]; then - continue; - fi - - # Set oldrev properly if this is branch creation. - if [ "${oldrev}" = "${zero_commit}" ]; then - oldrev="HEAD" - fi - - # Get list of files to look at using git diff - for file in $($GITCMD diff --stat --name-only --diff-filter=ACMRT ${oldrev}..${newrev}); do - # Get the size of this file - size=$($GITCMD cat-file -s ${newrev}:${file}) - # Check to see if for some reason we didn't get a size - if [ ! -z ${size} ]; then - # Compare filesize to MAXSIZE - if [ "${size}" -gt "${MAXSIZE}" ]; then - # Send output back to the user about oversized files. - echo "Error: ${file} larger than 5Mb. Please see Biocondcutor guidelines" - echo " https://bioconductor.org/developers/package-guidelines/" - # Fail here - exit 1 - fi # End size comparison - fi # End check for empty size - done # End list of files -done # End reading stdin - -# If successful, pass -exit 0 From 4f4af1526e2d7e44afe56e4d5ac1872f59c6224a Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Tue, 29 Aug 2017 17:47:18 -0400 Subject: [PATCH 16/70] check body of hook --- hooks/repo-specific/prevent_duplicate_commits.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hooks/repo-specific/prevent_duplicate_commits.py b/hooks/repo-specific/prevent_duplicate_commits.py index deb84a6..71f9770 100644 --- a/hooks/repo-specific/prevent_duplicate_commits.py +++ b/hooks/repo-specific/prevent_duplicate_commits.py @@ -2,7 +2,6 @@ import subprocess import sys -import fileinput # Global variables used by pre-recieve hook @@ -50,11 +49,18 @@ def prevent_duplicate_commits(oldrev, newrev, refname): # else git diff will report no diffs body1 = subprocess.check_output(["git", "show", "--format=%b", first]).strip() + if body1.startswith('git-svn-id'): + body1 = body1.splitlines()[1:] + ## If body1 is empty if not body1: continue body2 = subprocess.check_output(["git", "show", "--format=%b", second]).strip() + + if body2.startswith('git-svn-id'): + body2 = body2.splitlines()[1:] + ## if body2 is empty if not body2: continue From 0d4a27ae0fb150416fc830d152fae6e9011738ff Mon Sep 17 00:00:00 2001 From: Martin Morgan Date: Thu, 31 Aug 2017 05:23:14 -0400 Subject: [PATCH 17/70] update authz -> conf - allow '.' in data-experiment packages - update pre-receive hook on software packages --- R/authz_to_conf.R | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/R/authz_to_conf.R b/R/authz_to_conf.R index 4d109a9..744cb4a 100644 --- a/R/authz_to_conf.R +++ b/R/authz_to_conf.R @@ -13,13 +13,13 @@ fout <- "gitolite-admin/conf/packages.conf" stopifnot(file.exists(fin), file.exists(fin2), !file.exists(fout)) remap_re <- "[']" # re-map userid characters to '_' -group_re <- "^[-[:alnum:]]+ *= (.*)" +group_re <- "^[-.[:alnum:]]+ *= (.*)" software_template <- paste( "repo packages/%s", " RW master = %s", " RW RELEASE_3_5 = %s", - " option hook.pre-receive = disable-large-commits", + " option hook.pre-receive = pre-receive-hook", "", sep="\n" ) @@ -95,14 +95,14 @@ group_merge <- ## bioconductor.authz -repos_re <- "^\\[/trunk/madman/Rpacks/([[:alnum:]]+)]" +repos_re <- "^\\[/trunk/madman/Rpacks/([.[:alnum:]]+)]" reader_id <- "bioconductor-readers" writer_id <- "bioconductor-write0" svn_path <- "/bioconductor/trunk/madman/Rpacks" bioconductor_authz <- process_authz(fin, repos_re, reader_id, writer_id, svn_path) -repos_re <- "^\\[/trunk/experiment/pkgs/([[:alnum:]]+)]" +repos_re <- "^\\[/trunk/experiment/pkgs/([.[:alnum:]]+)]" reader_id <- "bioc-data_readers" writer_id <- "bioc-data-writers" svn_path <- "/bioc-data/trunk/experiment/pkgs" From b66b8cf28a11a4ca28994d6131919acc5f492d1b Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Thu, 31 Aug 2017 12:18:45 -0400 Subject: [PATCH 18/70] Pre-receive hook to just test for duplicate svn revisions --- hooks/repo-specific/pre-receive-hook | 2 +- .../prevent_duplicate_commits.py | 45 +++++++++++-------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/hooks/repo-specific/pre-receive-hook b/hooks/repo-specific/pre-receive-hook index fd30d26..dadf495 100644 --- a/hooks/repo-specific/pre-receive-hook +++ b/hooks/repo-specific/pre-receive-hook @@ -17,7 +17,7 @@ ZERO_COMMIT = "0000000000000000000000000000000000000000" # 3. create invalid commit history # 4. run python prevent-duplicate-commits from repository root # 5. create branch and repeat for each test -if False: +if True: refname=None revs = subprocess.check_output([ "git", "log", "-2", "--format=%H" diff --git a/hooks/repo-specific/prevent_duplicate_commits.py b/hooks/repo-specific/prevent_duplicate_commits.py index 71f9770..25310d6 100644 --- a/hooks/repo-specific/prevent_duplicate_commits.py +++ b/hooks/repo-specific/prevent_duplicate_commits.py @@ -2,6 +2,7 @@ import subprocess import sys +import re # Global variables used by pre-recieve hook @@ -28,6 +29,14 @@ to see body of commits. """ + +def get_revision(commit): + revision = '' + if commit.startswith("git-svn-id"): + revision = re.compile(".*git-svn-id: .*@([0-9]{6})").match(commit).group(1) + return revision + + def prevent_duplicate_commits(oldrev, newrev, refname): """Pre-receive hook to check for duplicate commits.""" try: @@ -49,26 +58,24 @@ def prevent_duplicate_commits(oldrev, newrev, refname): # else git diff will report no diffs body1 = subprocess.check_output(["git", "show", "--format=%b", first]).strip() - if body1.startswith('git-svn-id'): - body1 = body1.splitlines()[1:] - ## If body1 is empty - if not body1: - continue - body2 = subprocess.check_output(["git", "show", "--format=%b", second]).strip() - if body2.startswith('git-svn-id'): - body2 = body2.splitlines()[1:] - ## if body2 is empty - if not body2: - continue - - # Get diff of two commits - diff = subprocess.check_output(["git", "diff", first, second]) - # If the diff of two commits is empty, means they are the same. - # i.e duplicate - if not diff: - print(ERROR_DUPLICATE_COMMITS % (first, second)) - sys.exit(1) +# print("revision1: %s, commit: %s" +# % (get_revision(body1), first)) +# print("revision2: %s, commit: %s" +# % (get_revision(body2), second)) + rev1 = get_revision(body1) + rev2 = get_revision(body2) + if rev1 and rev2: + if rev1 == rev2: + # Get diff of two commits +# print("********body1:******** \n %s" % body1) +# print("********body2:******** \n %s" % body2) + diff = subprocess.check_output(["git", "diff", first, second]) + # If the diff of two commits is empty, means they are the same. + # i.e duplicate + if not diff: + print(ERROR_DUPLICATE_COMMITS % (first, second)) + sys.exit(1) return From 6345ed20889bf1ddfbab844e5f9a55ee5b8309d0 Mon Sep 17 00:00:00 2001 From: Martin Morgan Date: Thu, 31 Aug 2017 16:52:58 -0400 Subject: [PATCH 19/70] minor code tidy - Introduce GIT_COMMIT_LIST_LENGTH to reduce magic numbers - Introduce SVN_COMMIT_REGEX to compile once per scan - move `git show` to avoid code repetition - test for match rather than implicit double search for git-svn-id - opt for code formatting that allows args on one line - simplify condition testing -- rev1 and (rev1 == rev2) implies rev2 != None - remove dead code / comments --- .../prevent_duplicate_commits.py | 55 +++++++------------ 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/hooks/repo-specific/prevent_duplicate_commits.py b/hooks/repo-specific/prevent_duplicate_commits.py index 25310d6..b0376b7 100644 --- a/hooks/repo-specific/prevent_duplicate_commits.py +++ b/hooks/repo-specific/prevent_duplicate_commits.py @@ -6,6 +6,8 @@ # Global variables used by pre-recieve hook +GIT_COMMIT_LIST_LENGTH = "10" +SVN_COMMIT_REGEX = re.compile(".*git-svn-id: .*@([0-9]{6})") ZERO_COMMIT = "0000000000000000000000000000000000000000" ERROR_DUPLICATE_COMMITS = """Error: duplicate commits. @@ -29,53 +31,38 @@ to see body of commits. """ - -def get_revision(commit): - revision = '' - if commit.startswith("git-svn-id"): - revision = re.compile(".*git-svn-id: .*@([0-9]{6})").match(commit).group(1) +def get_svn_revision(commit): + body = subprocess.check_output([ "git", "show", "--format=%b", commit ]) + revision = SVN_COMMIT_REGEX.match(body) + if revision != None: + revision = revision.group(1) return revision def prevent_duplicate_commits(oldrev, newrev, refname): - """Pre-receive hook to check for duplicate commits.""" + """Pre-receive hook to check for duplicate SVN commits.""" try: - commit_list = subprocess.check_output(["git", - "rev-list", - newrev, "-n", "20"]) + commit_list = subprocess.check_output([ + "git", "rev-list", newrev, "-n", GIT_COMMIT_LIST_LENGTH + ]) except Exception as e: print("Exception: %s" % e) pass commit_list = commit_list.split("\n") commit_list = [item for item in commit_list if len(item)>0] - # For each of the first 10 pairs, check diff + # For each of the first GIT_COMMIT_LIST_LENGTH pairs, check diff for i in xrange(len(commit_list) - 1): first = commit_list[i] second = commit_list[i+1] - # use 'show' to test for empty commits, - # else git diff will report no diffs - body1 = subprocess.check_output(["git", "show", - "--format=%b", first]).strip() - body2 = subprocess.check_output(["git", "show", - "--format=%b", second]).strip() - -# print("revision1: %s, commit: %s" -# % (get_revision(body1), first)) -# print("revision2: %s, commit: %s" -# % (get_revision(body2), second)) - rev1 = get_revision(body1) - rev2 = get_revision(body2) - if rev1 and rev2: - if rev1 == rev2: - # Get diff of two commits -# print("********body1:******** \n %s" % body1) -# print("********body2:******** \n %s" % body2) - diff = subprocess.check_output(["git", "diff", first, second]) - # If the diff of two commits is empty, means they are the same. - # i.e duplicate - if not diff: - print(ERROR_DUPLICATE_COMMITS % (first, second)) - sys.exit(1) + rev1 = get_svn_revision(first) + rev2 = get_svn_revision(second) + if rev1 and (rev1 == rev2): + diff = subprocess.check_output(["git", "diff", first, second]) + # If the diff of two commits is empty, means they are the same. + # i.e duplicate + if not diff: + print(ERROR_DUPLICATE_COMMITS % (first, second)) + sys.exit(1) return From 9bb99b557983f4a07a295428bd5f01f4ef12c2c5 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Tue, 5 Sep 2017 10:30:58 -0400 Subject: [PATCH 20/70] Changes in prevent_large_files --- hooks/repo-specific/pre-receive-hook | 1 + hooks/repo-specific/prevent_large_files.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/hooks/repo-specific/pre-receive-hook b/hooks/repo-specific/pre-receive-hook index dadf495..eeb50fd 100644 --- a/hooks/repo-specific/pre-receive-hook +++ b/hooks/repo-specific/pre-receive-hook @@ -34,6 +34,7 @@ if __name__ == "__main__": oldrev, newrev, refname = std_input[0], std_input[1], std_input[2] # Check for zero commit, check branch deletions if newrev == ZERO_COMMIT: + print newrev continue # prevent large files prevent_large_files(oldrev, newrev, refname) diff --git a/hooks/repo-specific/prevent_large_files.py b/hooks/repo-specific/prevent_large_files.py index 5c17cd6..f5e374d 100644 --- a/hooks/repo-specific/prevent_large_files.py +++ b/hooks/repo-specific/prevent_large_files.py @@ -1,7 +1,5 @@ import subprocess import sys -import fileinput -import math # Global variables used by pre-recieve hook ZERO_COMMIT = "0000000000000000000000000000000000000000" @@ -19,8 +17,15 @@ def prevent_large_files(oldrev, newrev, refname): """Pre-receive hook to check for large files.""" # set oldrec properly if this is branch creation + print oldrev + print newrev if oldrev == ZERO_COMMIT: - oldrev = "HEAD" + output = subprocess.check_output(["git", "rev-list", + "--max-parents=0", + newrev]) + parent_commit = output.strip() + print parent_commit + # oldrev = "HEAD" list_files = subprocess.check_output(["git", "diff", "--name-only", "--diff-filter=ACMRT", From 55f8dbcd92a4d3b41a0f14ed03b3a185c6afbe16 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Tue, 5 Sep 2017 12:23:22 -0400 Subject: [PATCH 21/70] modified pre-recieve hooks --- hooks/repo-specific/disable-large-commits | 42 ++++++++++++++++++++++ hooks/repo-specific/pre-receive-hook | 5 ++- hooks/repo-specific/prevent_large_files.py | 22 ++++++------ 3 files changed, 54 insertions(+), 15 deletions(-) create mode 100755 hooks/repo-specific/disable-large-commits diff --git a/hooks/repo-specific/disable-large-commits b/hooks/repo-specific/disable-large-commits new file mode 100755 index 0000000..d752dad --- /dev/null +++ b/hooks/repo-specific/disable-large-commits @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# +# Pre-receive hook that will block any new commits that contain files larger than 5Mb in size. +# + +GITCMD="/usr/bin/git" +zero_commit="0000000000000000000000000000000000000000" +MAXSIZE="5000000" # 5MB limit on file size + +# Read stdin for ref information +while read oldrev newrev refname; do + # Skip branch deletions + if [ "${newrev}" = "${zero_commit}" ]; then + continue; + fi + + # Set oldrev properly if this is branch creation. + if [ "${oldrev}" = "${zero_commit}" ]; then + oldrev="HEAD" + fi + + # Get list of files to look at using git diff + for file in $($GITCMD diff --stat --name-only --diff-filter=ACMRT ${oldrev}..${newrev}); do + # Get the size of this file + size=$($GITCMD cat-file -s ${newrev}:${file}) + # Check to see if for some reason we didn't get a size + if [ ! -z ${size} ]; then + # Compare filesize to MAXSIZE + if [ "${size}" -gt "${MAXSIZE}" ]; then + # Send output back to the user about oversized files. + echo "Error: ${file} larger than 5Mb. Please see Biocondcutor guidelines" + echo " https://bioconductor.org/developers/package-guidelines/" + # Fail here + exit 1 + fi # End size comparison + fi # End check for empty size + done # End list of files +done # End reading stdin + +# If successful, pass +exit 0 diff --git a/hooks/repo-specific/pre-receive-hook b/hooks/repo-specific/pre-receive-hook index eeb50fd..4592206 100644 --- a/hooks/repo-specific/pre-receive-hook +++ b/hooks/repo-specific/pre-receive-hook @@ -17,7 +17,7 @@ ZERO_COMMIT = "0000000000000000000000000000000000000000" # 3. create invalid commit history # 4. run python prevent-duplicate-commits from repository root # 5. create branch and repeat for each test -if True: +if False: refname=None revs = subprocess.check_output([ "git", "log", "-2", "--format=%H" @@ -31,10 +31,9 @@ if True: if __name__ == "__main__": for line in fileinput.input(): std_input = line.split(" ") - oldrev, newrev, refname = std_input[0], std_input[1], std_input[2] + oldrev, newrev, refname = [elt.strip() for elt in std_input] # Check for zero commit, check branch deletions if newrev == ZERO_COMMIT: - print newrev continue # prevent large files prevent_large_files(oldrev, newrev, refname) diff --git a/hooks/repo-specific/prevent_large_files.py b/hooks/repo-specific/prevent_large_files.py index f5e374d..d6f0e75 100644 --- a/hooks/repo-specific/prevent_large_files.py +++ b/hooks/repo-specific/prevent_large_files.py @@ -16,24 +16,22 @@ def prevent_large_files(oldrev, newrev, refname): """Pre-receive hook to check for large files.""" - # set oldrec properly if this is branch creation - print oldrev - print newrev + # set oldrev properly if this is branch creation if oldrev == ZERO_COMMIT: - output = subprocess.check_output(["git", "rev-list", - "--max-parents=0", - newrev]) - parent_commit = output.strip() - print parent_commit - # oldrev = "HEAD" + if refname == "refs/heads/master": + oldrev = subprocess.check_output(["git", "rev-list", + "--max-parents=0", + newrev]).strip() + else: + oldrev = "HEAD" list_files = subprocess.check_output(["git", "diff", - "--name-only", "--diff-filter=ACMRT", - oldrev + ".." + newrev]) + "--name-only", "--diff-filter=ACMRT", + oldrev + ".." + newrev]) for fl in list_files.splitlines(): size = subprocess.check_output(["git", "cat-file", "-s", - newrev + ":" + fl]) + newrev + ":" + fl]) # Check to see if for some reason we didn't get a size size = int(size.strip()) if size: From 646f474e6a4f0b68249fedb05c70f56dd3961acd Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Mon, 11 Sep 2017 15:06:54 -0400 Subject: [PATCH 22/70] README --- doc/README.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 doc/README.md diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 0000000..ff884a3 --- /dev/null +++ b/doc/README.md @@ -0,0 +1,6 @@ +The contents of the documentation are most up to date at + + http://bioconductor.org/developers/how-to/git/ + + + From b2c2436d333a927d2fdc28585a80da65082df7fb Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Fri, 22 Sep 2017 11:19:31 -0400 Subject: [PATCH 23/70] update hooks --- hooks/repo-specific/pre-receive-hook | 0 hooks/repo-specific/prevent_large_files.py | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 hooks/repo-specific/pre-receive-hook diff --git a/hooks/repo-specific/pre-receive-hook b/hooks/repo-specific/pre-receive-hook old mode 100644 new mode 100755 diff --git a/hooks/repo-specific/prevent_large_files.py b/hooks/repo-specific/prevent_large_files.py index d6f0e75..03fd918 100644 --- a/hooks/repo-specific/prevent_large_files.py +++ b/hooks/repo-specific/prevent_large_files.py @@ -19,9 +19,9 @@ def prevent_large_files(oldrev, newrev, refname): # set oldrev properly if this is branch creation if oldrev == ZERO_COMMIT: if refname == "refs/heads/master": - oldrev = subprocess.check_output(["git", "rev-list", - "--max-parents=0", - newrev]).strip() + oldrev = subprocess.check_output([ + "git", "rev-list", "--max-parents=0", newrev + ]).split().pop().strip() else: oldrev = "HEAD" From 76357e1fff6fdd829d8047e6a26da37f17114d2e Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Tue, 26 Sep 2017 10:37:57 -0400 Subject: [PATCH 24/70] new folder 'common' for post receive hooks --- hooks/common/feed.xml | 4 ++++ hooks/common/post-receive-python | 40 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 hooks/common/feed.xml create mode 100755 hooks/common/post-receive-python diff --git a/hooks/common/feed.xml b/hooks/common/feed.xml new file mode 100644 index 0000000..b0d1ec6 --- /dev/null +++ b/hooks/common/feed.xml @@ -0,0 +1,4 @@ + + +Bioconductor GIT feed + diff --git a/hooks/common/post-receive-python b/hooks/common/post-receive-python new file mode 100755 index 0000000..5c862e1 --- /dev/null +++ b/hooks/common/post-receive-python @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +import fileinput +import subprocess +import datetime + +# Path to feed.xml +FEED="/Users/ni41435_ca/Documents/bioc_git_transition/hooks/common/feed.xml" + +def rss_feed(oldrev, newrev, refname): + """Pre-receive hook to check for duplicate SVN commits.""" + try: + latest_commit = subprocess.check_output([ + "git", "log", oldrev + ".." + newrev, + "--pretty=format:'%h|%an|%s|%at'" + ]) + except Exception as e: + print("Exception: %s" % e) + pass + if latest_commit: + commit_id, author, message, timestamp = latest_commit.split("|") + date = datetime.datetime.fromtimestamp( + int(timestamp)).strftime('%Y-%m-%d %H:%M:%S') + feed = "\\n\ + \\%s\<\/name\>\<\/author\>\n\ + \%s\<\/title\>\n\ + \%s\<\/published\>\n\ + \\n\ + \\n\ + \<\/summary\>\n\ + \<\/entry\>" % (author, message, date) + # Not sure why we add the commits variable after the 3rd line in XML file + return feed + + +for line in fileinput.input(): + std_input = line.split(" ") + oldrev, newrev, refname = [item.strip() for item in std_input] + rss_feed(oldrev, newrev, refname) From 2a0bb426acc122592b8215e52eb247bbed0a487f Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Fri, 13 Oct 2017 10:52:02 -0400 Subject: [PATCH 25/70] changes to post-revice python' --- hooks/common/feed.xml | 12 +++++++- hooks/common/post-receive-python | 48 ++++++++++++++++++++------------ 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/hooks/common/feed.xml b/hooks/common/feed.xml index b0d1ec6..ebbd074 100644 --- a/hooks/common/feed.xml +++ b/hooks/common/feed.xml @@ -1,4 +1,14 @@ Bioconductor GIT feed - + + d83552d + Nitesh Turaga + Change + 2017-09-26 10:11:17 + + + + + \ No newline at end of file diff --git a/hooks/common/post-receive-python b/hooks/common/post-receive-python index 5c862e1..2658ce5 100755 --- a/hooks/common/post-receive-python +++ b/hooks/common/post-receive-python @@ -5,36 +5,48 @@ import subprocess import datetime # Path to feed.xml -FEED="/Users/ni41435_ca/Documents/bioc_git_transition/hooks/common/feed.xml" +fpath="/Users/ni41435_ca/Documents/bioc_git_transition/hooks/common/feed.xml" + +FEED=""" + +Bioconductor GIT feed +%s +""" + def rss_feed(oldrev, newrev, refname): """Pre-receive hook to check for duplicate SVN commits.""" try: latest_commit = subprocess.check_output([ "git", "log", oldrev + ".." + newrev, - "--pretty=format:'%h|%an|%s|%at'" + "--pretty=format:%h|%an|%s|%at" ]) except Exception as e: print("Exception: %s" % e) pass if latest_commit: commit_id, author, message, timestamp = latest_commit.split("|") - date = datetime.datetime.fromtimestamp( - int(timestamp)).strftime('%Y-%m-%d %H:%M:%S') - feed = "\\n\ - \\%s\<\/name\>\<\/author\>\n\ - \%s\<\/title\>\n\ - \%s\<\/published\>\n\ - \\n\ - \\n\ - \<\/summary\>\n\ - \<\/entry\>" % (author, message, date) + date = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y-%m-%d %H:%M:%S') + entry = "\n\ + %s\n\ + %s\n\ + %s\n\ + %s\n\ + \n\ + \n\ + \n\ + " % (commit_id, author, message, date) # Not sure why we add the commits variable after the 3rd line in XML file - return feed + new_feed = FEED % entry + with open(fpath, "w") as f: + f.write(new_feed) + return + + +#for line in fileinput.input(): +# std_input = line.split(" ") +# oldrev, newrev, refname = [item.strip() for item in std_input] +rss_feed("67ccbb7b161b5736cb0202a2a783b32e68c689dc", "d83552d75588957a54222c010e8c1311cac29464","refs/head/master") -for line in fileinput.input(): - std_input = line.split(" ") - oldrev, newrev, refname = [item.strip() for item in std_input] - rss_feed(oldrev, newrev, refname) From 10c37cb2a31f83ed114a112f62d231f1c1f87a7e Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Thu, 9 Nov 2017 13:18:57 -0500 Subject: [PATCH 26/70] post-receive in shell --- hooks/common/post-receive | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100755 hooks/common/post-receive diff --git a/hooks/common/post-receive b/hooks/common/post-receive new file mode 100755 index 0000000..488f60f --- /dev/null +++ b/hooks/common/post-receive @@ -0,0 +1,34 @@ +#!/bin/sh +# +# This script adds every pushed commit info to XML file +# Make sure to place it in your bare repo's /hooks dir and make it executable +# + +git update-server-info + +FILE="/Users/ni41435_ca/Documents/bioc_git_transition/hooks/common/feed.xml" + +# Get diapasone of pushed commits +while read oldrev newrev refname +do + # Loop over pushed commits and concatenate XML data into $commits variable + commits=$(git log $oldrev..$newrev --pretty=format:'%h|%an|%s|%at' | + while IFS='|' read hash author message timestamp + do + + date=`date -d @$timestamp +%Y-%m-%dT%H:%M:%SZ` + echo "\\n\ + \\$author\<\/name\>\<\/author\>\n\ + \$message\<\/title\>\n\ + \$date\<\/published\>\n\ + \\n\ + \\n\ + \<\/summary\>\n\ + \<\/entry\>" + + done) + + # Add $commits variable value after 3rd line in XML file + sed -i "3 a $(echo $commits)" $FILE +done From 8bf24e4d42da71d395b1910ebb0c20cb6a5c2eeb Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Wed, 29 Nov 2017 13:58:51 -0500 Subject: [PATCH 27/70] rss_feed.py updates --- hooks/common/feed.xml | 12 +----- hooks/common/post-receive-hook | 16 +++++++ hooks/common/post-receive-python | 52 ----------------------- hooks/common/rss_feed.py | 73 ++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 63 deletions(-) create mode 100755 hooks/common/post-receive-hook delete mode 100755 hooks/common/post-receive-python create mode 100644 hooks/common/rss_feed.py diff --git a/hooks/common/feed.xml b/hooks/common/feed.xml index ebbd074..b0d1ec6 100644 --- a/hooks/common/feed.xml +++ b/hooks/common/feed.xml @@ -1,14 +1,4 @@ Bioconductor GIT feed - - d83552d - Nitesh Turaga - Change - 2017-09-26 10:11:17 - - - - - \ No newline at end of file + diff --git a/hooks/common/post-receive-hook b/hooks/common/post-receive-hook new file mode 100755 index 0000000..96f53e0 --- /dev/null +++ b/hooks/common/post-receive-hook @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +import fileinput +from rss_feed import rss_feed + + +if __name__ == "__main__": + # Path to feed.xml + fpath="feed.xml" + # Run function for RSS feed + for line in fileinput.input(): + std_input = line.split(" ") + oldrev, newrev, refname = [item.strip() for item in std_input] + + # RSS-feed post-receive hook + rss_feed(oldrev, newrev, refname, fpath) diff --git a/hooks/common/post-receive-python b/hooks/common/post-receive-python deleted file mode 100755 index 2658ce5..0000000 --- a/hooks/common/post-receive-python +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python - -import fileinput -import subprocess -import datetime - -# Path to feed.xml -fpath="/Users/ni41435_ca/Documents/bioc_git_transition/hooks/common/feed.xml" - -FEED=""" - -Bioconductor GIT feed -%s -""" - - -def rss_feed(oldrev, newrev, refname): - """Pre-receive hook to check for duplicate SVN commits.""" - try: - latest_commit = subprocess.check_output([ - "git", "log", oldrev + ".." + newrev, - "--pretty=format:%h|%an|%s|%at" - ]) - except Exception as e: - print("Exception: %s" % e) - pass - if latest_commit: - commit_id, author, message, timestamp = latest_commit.split("|") - date = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y-%m-%d %H:%M:%S') - entry = "\n\ - %s\n\ - %s\n\ - %s\n\ - %s\n\ - \n\ - \n\ - \n\ - " % (commit_id, author, message, date) - # Not sure why we add the commits variable after the 3rd line in XML file - new_feed = FEED % entry - with open(fpath, "w") as f: - f.write(new_feed) - return - - -#for line in fileinput.input(): -# std_input = line.split(" ") -# oldrev, newrev, refname = [item.strip() for item in std_input] - -rss_feed("67ccbb7b161b5736cb0202a2a783b32e68c689dc", "d83552d75588957a54222c010e8c1311cac29464","refs/head/master") - diff --git a/hooks/common/rss_feed.py b/hooks/common/rss_feed.py new file mode 100644 index 0000000..f0db1e9 --- /dev/null +++ b/hooks/common/rss_feed.py @@ -0,0 +1,73 @@ +import subprocess +import datetime +import time +import re +from xml.dom.minidom import parseString + + +def limit_feed_length(fpath, length): + """ This is run only once every day""" + with open(fpath, "r") as f: + data = f.read() + dom = parseString(data) + if len(dom.getElementsByTagName('entry')) > length: + # If more than length get all elements at the end + last = dom.getElementsByTagName('entry')[length:] + dom.documentElement.removeChild(last) + dom.writexml(open(fpath,"w")) + return + + +def write_feed(entry, fpath): + """Write feed to the beginning of the file""" + with open(fpath, "r+") as f: + text = f.read() + text = re.sub("feed\n", + "feed\n" + entry, + text) + f.seek(0) + f.write(text) + return + +def rss_feed(oldrev, newrev, refname, fpath, length): + """Post receive hook to check start Git RSS feed""" + try: + latest_commit = subprocess.check_output([ + "git", "log", oldrev + ".." + newrev, + "--pretty=format:%h|%an|%s|%at" + ]) + except Exception as e: + print("Exception: %s" % e) + pass + if latest_commit: + commit_id, author, message, timestamp = latest_commit.split("|") + date = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y-%m-%d %H:%M:%S') + entry = "\n\ + %s\n\ + %s\n\ + %s\n\ + %s\n\ + \n\ + \n\ + \n\ + \n" % (commit_id, author, message, date) + ## Write FEED and sleep to avoid race condition + try: + write_feed(entry, fpath) + except IOError as e: + ## Avoid race condition during file write + time.sleep(2) + try: + write_feed(entry, fpath) + except IOError as e: + print("Error writing feed", e) + + ## Limit feed length to 200 + try: + limit_feed_length(fpath, length) + except Exception as e: + print("Error limiting feed size", e) + return + + From 45230b529351cd2f34c1b270d6ff8ef9c1f22baa Mon Sep 17 00:00:00 2001 From: vobencha Date: Thu, 7 Dec 2017 19:29:52 -0800 Subject: [PATCH 28/70] fix typo --- admin-docs/configure.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin-docs/configure.md b/admin-docs/configure.md index 86f93d6..859649e 100644 --- a/admin-docs/configure.md +++ b/admin-docs/configure.md @@ -213,7 +213,7 @@ out-of-the-box Apache configuration to limit what users can see. ## Push / pull access via SSH - + ### svn 'authz' to gitolite 'conf' The gitolite configuration involves From 1cd93445f3ba40714bb904e93f847af55e73836c Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Fri, 8 Dec 2017 13:09:17 -0500 Subject: [PATCH 29/70] Find unknown commits --- admin-docs/find_unknowns.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 admin-docs/find_unknowns.sh diff --git a/admin-docs/find_unknowns.sh b/admin-docs/find_unknowns.sh new file mode 100644 index 0000000..bd8fc26 --- /dev/null +++ b/admin-docs/find_unknowns.sh @@ -0,0 +1,9 @@ +for pkg in /home/git/repositories/packages/*git; +do { + cd $pkg + unknown=`git log --all --oneline --committer="unknown"` + latest=`git log --all --since="2017-08-15"` + if [ ! -z "$unknown" ] && [ -z "$latest" ]; then + echo `basename $pkg`; + fi +} done From 71a7a4089062e9265a6b1d027fb69de35f2992c4 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Mon, 18 Dec 2017 10:19:45 -0500 Subject: [PATCH 30/70] Add rss feed post receive hook --- hooks/common/feed.xml | 4 - hooks/common/post-receive | 34 ------- hooks/common/post-receive-hook | 16 ---- hooks/common/rss_feed.py | 73 --------------- hooks/repo-specific/post-receive-hook | 123 ++++++++++++++++++++++++++ hooks/repo-specific/rss_feed.py | 52 +++++++++++ 6 files changed, 175 insertions(+), 127 deletions(-) delete mode 100644 hooks/common/feed.xml delete mode 100755 hooks/common/post-receive delete mode 100755 hooks/common/post-receive-hook delete mode 100644 hooks/common/rss_feed.py create mode 100755 hooks/repo-specific/post-receive-hook create mode 100644 hooks/repo-specific/rss_feed.py diff --git a/hooks/common/feed.xml b/hooks/common/feed.xml deleted file mode 100644 index b0d1ec6..0000000 --- a/hooks/common/feed.xml +++ /dev/null @@ -1,4 +0,0 @@ - - -Bioconductor GIT feed - diff --git a/hooks/common/post-receive b/hooks/common/post-receive deleted file mode 100755 index 488f60f..0000000 --- a/hooks/common/post-receive +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -# -# This script adds every pushed commit info to XML file -# Make sure to place it in your bare repo's /hooks dir and make it executable -# - -git update-server-info - -FILE="/Users/ni41435_ca/Documents/bioc_git_transition/hooks/common/feed.xml" - -# Get diapasone of pushed commits -while read oldrev newrev refname -do - # Loop over pushed commits and concatenate XML data into $commits variable - commits=$(git log $oldrev..$newrev --pretty=format:'%h|%an|%s|%at' | - while IFS='|' read hash author message timestamp - do - - date=`date -d @$timestamp +%Y-%m-%dT%H:%M:%SZ` - echo "\\n\ - \\$author\<\/name\>\<\/author\>\n\ - \$message\<\/title\>\n\ - \$date\<\/published\>\n\ - \\n\ - \\n\ - \<\/summary\>\n\ - \<\/entry\>" - - done) - - # Add $commits variable value after 3rd line in XML file - sed -i "3 a $(echo $commits)" $FILE -done diff --git a/hooks/common/post-receive-hook b/hooks/common/post-receive-hook deleted file mode 100755 index 96f53e0..0000000 --- a/hooks/common/post-receive-hook +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python - -import fileinput -from rss_feed import rss_feed - - -if __name__ == "__main__": - # Path to feed.xml - fpath="feed.xml" - # Run function for RSS feed - for line in fileinput.input(): - std_input = line.split(" ") - oldrev, newrev, refname = [item.strip() for item in std_input] - - # RSS-feed post-receive hook - rss_feed(oldrev, newrev, refname, fpath) diff --git a/hooks/common/rss_feed.py b/hooks/common/rss_feed.py deleted file mode 100644 index f0db1e9..0000000 --- a/hooks/common/rss_feed.py +++ /dev/null @@ -1,73 +0,0 @@ -import subprocess -import datetime -import time -import re -from xml.dom.minidom import parseString - - -def limit_feed_length(fpath, length): - """ This is run only once every day""" - with open(fpath, "r") as f: - data = f.read() - dom = parseString(data) - if len(dom.getElementsByTagName('entry')) > length: - # If more than length get all elements at the end - last = dom.getElementsByTagName('entry')[length:] - dom.documentElement.removeChild(last) - dom.writexml(open(fpath,"w")) - return - - -def write_feed(entry, fpath): - """Write feed to the beginning of the file""" - with open(fpath, "r+") as f: - text = f.read() - text = re.sub("feed\n", - "feed\n" + entry, - text) - f.seek(0) - f.write(text) - return - -def rss_feed(oldrev, newrev, refname, fpath, length): - """Post receive hook to check start Git RSS feed""" - try: - latest_commit = subprocess.check_output([ - "git", "log", oldrev + ".." + newrev, - "--pretty=format:%h|%an|%s|%at" - ]) - except Exception as e: - print("Exception: %s" % e) - pass - if latest_commit: - commit_id, author, message, timestamp = latest_commit.split("|") - date = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y-%m-%d %H:%M:%S') - entry = "\n\ - %s\n\ - %s\n\ - %s\n\ - %s\n\ - \n\ - \n\ - \n\ - \n" % (commit_id, author, message, date) - ## Write FEED and sleep to avoid race condition - try: - write_feed(entry, fpath) - except IOError as e: - ## Avoid race condition during file write - time.sleep(2) - try: - write_feed(entry, fpath) - except IOError as e: - print("Error writing feed", e) - - ## Limit feed length to 200 - try: - limit_feed_length(fpath, length) - except Exception as e: - print("Error limiting feed size", e) - return - - diff --git a/hooks/repo-specific/post-receive-hook b/hooks/repo-specific/post-receive-hook new file mode 100755 index 0000000..26ece8a --- /dev/null +++ b/hooks/repo-specific/post-receive-hook @@ -0,0 +1,123 @@ +#!/usr/bin/env python + +import fileinput +from rss_feed import rss_feed +from xml.etree.ElementTree import parse, fromstring +import subprocess +import fcntl +import sys +import logging +logging.basicConfig(filename='/tmp/post-recieve.log', level=logging.DEBUG) + +ZERO_COMMIT = "0000000000000000000000000000000000000000" +BASE_PATH = "/home/git/rss/" + + +def indent(elem, level=0): + i = "\n" + level*" " + if len(elem): + if not elem.text or not elem.text.strip(): + elem.text = i + " " + if not elem.tail or not elem.tail.strip(): + elem.tail = i + for elem in elem: + indent(elem, level+1) + if not elem.tail or not elem.tail.strip(): + elem.tail = i + else: + if level and (not elem.tail or not elem.tail.strip()): + elem.tail = i + + +def write_and_limit_feed(entry_list, length, feed): + doc = parse(feed) + root = doc.getroot() + + # Get items + channel_root = root.find("channel") + items = channel_root.findall("item") + # Write feed + for entry in entry_list: + # 5 is the entry position in the feed + channel_root.insert(5, entry) + # Remove extra elements + if len(items) > length: + extra_items = items[length:] + for extra_item in extra_items: + channel_root.remove(extra_item) + indent(channel_root) + feed.seek(0) + feed.truncate() + doc.write(feed) + feed.write("\n") + feed.flush() + return feed + + +if False: + fh = "/tmp/gitlog.xml" + feed = open(fh, "r+") + refname = None + revs = subprocess.check_output([ + "git", "log", "-2", "--format=%H" + ]).splitlines() + newrev = revs[0].strip() + oldrev = revs[1].strip() + rss_feed(oldrev, newrev, refname, 5) + sample_entry = """ + + 2309fc133512c4e25d8942c3d0ae6fc198bf9ba9 + https://www.bioconductor.org + 4: test + Nitesh + 2017-12-08 17:26:18 + + """ + entry = fromstring(sample_entry) + write_and_limit_feed([entry], 5, fh) + fh.close() + sys.exit(0) + + +if __name__ == "__main__": + # Path to feed.xml + fpath = BASE_PATH + "gitlog.xml" + fpath_release = BASE_PATH + "gitlog.release.xml" + length = 199 + + # Run function for RSS feed + feed = open(fpath, "r+") + feed_release = open(fpath_release, 'r+') + + # Obtain a lock + fcntl.lockf(feed, fcntl.LOCK_EX) + fcntl.lockf(feed_release, fcntl.LOCK_EX) + + for line in fileinput.input(): + std_input = line.split(" ") + oldrev, newrev, refname = [item.strip() for item in std_input] + # Check for zero commit, check branch deletions + # also, avoid new package additions + if (oldrev == ZERO_COMMIT or newrev == ZERO_COMMIT): + continue + # Split feed into correct files + if ("RELEASE" in refname): + # RSS-feed post-receive hook + entry = rss_feed(oldrev, newrev, refname, length) + write_and_limit_feed(entry, length, feed_release) + else: + entry = rss_feed(oldrev, newrev, refname, length) + write_and_limit_feed(entry, length, feed) + + cmd = ['scp', 'gitlog.xml', 'gitlog.release.xml', + 'biocadmin@staging.bioconductor.org:/home/biocadmin/bioc-test-web/bioconductor.org/assets/developers/rss-feeds/.'] + subprocess.check_call(cmd, cwd=BASE_PATH) + + # Release the lock + fcntl.lockf(feed, fcntl.LOCK_UN) + fcntl.lockf(feed_release, fcntl.LOCK_UN) + feed.close() + feed_release.close() + + logging.info("Exit after removing locks") + diff --git a/hooks/repo-specific/rss_feed.py b/hooks/repo-specific/rss_feed.py new file mode 100644 index 0000000..fdaf92a --- /dev/null +++ b/hooks/repo-specific/rss_feed.py @@ -0,0 +1,52 @@ +import subprocess +import datetime +# import re +from os.path import basename, abspath +from xml.etree.ElementTree import fromstring +import logging + + +ENTRY=""" + + %s + https://bioconductor.org + %s + %s + %s + %s + +""" + + +def rss_feed(oldrev, newrev, refname, length): + """Post receive hook to check start Git RSS feed""" + entry_list = [] + try: + latest_commit = subprocess.check_output([ + "git", "log", oldrev + ".." + newrev, + "--pretty=format:%H|%an|%s|%at" + ]) + # Get package name + package_path = subprocess.check_output([ + "git", "rev-parse", "--show-toplevel"]).strip() + package_name = basename(abspath(package_path)).replace(".git", "") + except Exception as e: + logging.error("Exception: %s" % e) + pass + if latest_commit: + # If more than one commit to unpack + latest_commit = latest_commit.split("\n") + # Reverse if there are multiple commits + for commit in latest_commit[::-1]: + commit_id, author, commit_msg, timestamp = commit.split("|") + pubDate = datetime.datetime.fromtimestamp( + float(timestamp)).strftime('%Y-%m-%d %H:%M:%S') + + entry = ENTRY % (package_name, + commit_msg, + author, + pubDate, + commit_id) + # Add entry as element in xml.etree + entry_list.append(fromstring(entry)) + return entry_list From 2d44f793a03fcfb4637f582690b2c7144a5100ad Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Wed, 10 Jan 2018 09:51:21 -0500 Subject: [PATCH 31/70] Modify hooks to reflect latest changes --- hooks/repo-specific/post-receive-hook | 27 ++++++++++--------- .../prevent_duplicate_commits.py | 2 +- hooks/repo-specific/rss_feed.py | 21 ++++++++++----- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/hooks/repo-specific/post-receive-hook b/hooks/repo-specific/post-receive-hook index 26ece8a..e69eb50 100755 --- a/hooks/repo-specific/post-receive-hook +++ b/hooks/repo-specific/post-receive-hook @@ -68,7 +68,9 @@ if False: 2309fc133512c4e25d8942c3d0ae6fc198bf9ba9 https://www.bioconductor.org - 4: test + Nitesh 2017-12-08 17:26:18 @@ -83,7 +85,7 @@ if __name__ == "__main__": # Path to feed.xml fpath = BASE_PATH + "gitlog.xml" fpath_release = BASE_PATH + "gitlog.release.xml" - length = 199 + length = 499 # Run function for RSS feed feed = open(fpath, "r+") @@ -91,7 +93,6 @@ if __name__ == "__main__": # Obtain a lock fcntl.lockf(feed, fcntl.LOCK_EX) - fcntl.lockf(feed_release, fcntl.LOCK_EX) for line in fileinput.input(): std_input = line.split(" ") @@ -101,21 +102,23 @@ if __name__ == "__main__": if (oldrev == ZERO_COMMIT or newrev == ZERO_COMMIT): continue # Split feed into correct files - if ("RELEASE" in refname): - # RSS-feed post-receive hook - entry = rss_feed(oldrev, newrev, refname, length) - write_and_limit_feed(entry, length, feed_release) - else: - entry = rss_feed(oldrev, newrev, refname, length) - write_and_limit_feed(entry, length, feed) - + try: + if ("RELEASE" in refname): + # RSS-feed post-receive hook + entry = rss_feed(oldrev, newrev, refname, length) + write_and_limit_feed(entry, length, feed_release) + else: + entry = rss_feed(oldrev, newrev, refname, length) + write_and_limit_feed(entry, length, feed) + except Exception as e: + print("Note: failed to update RSS feed; git repository updated successfully.") + logging.error(e) cmd = ['scp', 'gitlog.xml', 'gitlog.release.xml', 'biocadmin@staging.bioconductor.org:/home/biocadmin/bioc-test-web/bioconductor.org/assets/developers/rss-feeds/.'] subprocess.check_call(cmd, cwd=BASE_PATH) # Release the lock fcntl.lockf(feed, fcntl.LOCK_UN) - fcntl.lockf(feed_release, fcntl.LOCK_UN) feed.close() feed_release.close() diff --git a/hooks/repo-specific/prevent_duplicate_commits.py b/hooks/repo-specific/prevent_duplicate_commits.py index b0376b7..7418f90 100644 --- a/hooks/repo-specific/prevent_duplicate_commits.py +++ b/hooks/repo-specific/prevent_duplicate_commits.py @@ -6,7 +6,7 @@ # Global variables used by pre-recieve hook -GIT_COMMIT_LIST_LENGTH = "10" +GIT_COMMIT_LIST_LENGTH = "50" SVN_COMMIT_REGEX = re.compile(".*git-svn-id: .*@([0-9]{6})") ZERO_COMMIT = "0000000000000000000000000000000000000000" ERROR_DUPLICATE_COMMITS = """Error: duplicate commits. diff --git a/hooks/repo-specific/rss_feed.py b/hooks/repo-specific/rss_feed.py index fdaf92a..9ebf07c 100644 --- a/hooks/repo-specific/rss_feed.py +++ b/hooks/repo-specific/rss_feed.py @@ -9,9 +9,9 @@ ENTRY=""" %s - https://bioconductor.org - %s - %s + https://bioconductor.org/packages/%s/ + + %s %s @@ -24,7 +24,7 @@ def rss_feed(oldrev, newrev, refname, length): try: latest_commit = subprocess.check_output([ "git", "log", oldrev + ".." + newrev, - "--pretty=format:%H|%an|%s|%at" + "--pretty=format:%H|%an|%ae|%at" ]) # Get package name package_path = subprocess.check_output([ @@ -38,13 +38,20 @@ def rss_feed(oldrev, newrev, refname, length): latest_commit = latest_commit.split("\n") # Reverse if there are multiple commits for commit in latest_commit[::-1]: - commit_id, author, commit_msg, timestamp = commit.split("|") + commit_id, author, email, timestamp = commit.split("|") pubDate = datetime.datetime.fromtimestamp( float(timestamp)).strftime('%Y-%m-%d %H:%M:%S') - + commit_msg = subprocess.check_output(["git", "log" , + "--pretty=format:%B", + "-n", "1", commit_id]) + if "RELEASE" in refname: + link = package_name + else: + link = "devel/" + package_name entry = ENTRY % (package_name, + link, commit_msg, - author, + author + " <" + email + ">", pubDate, commit_id) # Add entry as element in xml.etree From d023bcd76c2441825426385b5caf363cda30ebbc Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Mon, 26 Feb 2018 10:20:09 -0500 Subject: [PATCH 32/70] rss feed ISO --- hooks/repo-specific/rss_feed.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hooks/repo-specific/rss_feed.py b/hooks/repo-specific/rss_feed.py index 9ebf07c..5aabf41 100644 --- a/hooks/repo-specific/rss_feed.py +++ b/hooks/repo-specific/rss_feed.py @@ -24,7 +24,7 @@ def rss_feed(oldrev, newrev, refname, length): try: latest_commit = subprocess.check_output([ "git", "log", oldrev + ".." + newrev, - "--pretty=format:%H|%an|%ae|%at" + "--pretty=format:%H|%an|%ae|%ai" ]) # Get package name package_path = subprocess.check_output([ @@ -39,8 +39,8 @@ def rss_feed(oldrev, newrev, refname, length): # Reverse if there are multiple commits for commit in latest_commit[::-1]: commit_id, author, email, timestamp = commit.split("|") - pubDate = datetime.datetime.fromtimestamp( - float(timestamp)).strftime('%Y-%m-%d %H:%M:%S') + #pubDate = datetime.datetime.fromtimestamp( + # float(timestamp)).strftime('%Y-%m-%d %H:%M:%S') commit_msg = subprocess.check_output(["git", "log" , "--pretty=format:%B", "-n", "1", commit_id]) @@ -52,8 +52,9 @@ def rss_feed(oldrev, newrev, refname, length): link, commit_msg, author + " <" + email + ">", - pubDate, + timestamp, commit_id) # Add entry as element in xml.etree entry_list.append(fromstring(entry)) return entry_list + From a1c273389c518b3551ae405e112b407676eb28cd Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Wed, 28 Mar 2018 14:03:51 -0400 Subject: [PATCH 33/70] remove unused code --- hooks/repo-specific/rss_feed.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/hooks/repo-specific/rss_feed.py b/hooks/repo-specific/rss_feed.py index 5aabf41..27b67c9 100644 --- a/hooks/repo-specific/rss_feed.py +++ b/hooks/repo-specific/rss_feed.py @@ -1,6 +1,4 @@ import subprocess -import datetime -# import re from os.path import basename, abspath from xml.etree.ElementTree import fromstring import logging From 7c727629e087c54b10b512afaa3dd1ab75556968 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Fri, 30 Mar 2018 13:43:55 -0400 Subject: [PATCH 34/70] Add pre-receive hook to prevent bad version numbers, add test cases as well --- .../prevent_bad_version_numbers.py | 119 ++++++++++++++++++ .../test_prevent_bad_version_numbers.py | 113 +++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 hooks/repo-specific/prevent_bad_version_numbers.py create mode 100644 hooks/repo-specific/test_prevent_bad_version_numbers.py diff --git a/hooks/repo-specific/prevent_bad_version_numbers.py b/hooks/repo-specific/prevent_bad_version_numbers.py new file mode 100644 index 0000000..f675918 --- /dev/null +++ b/hooks/repo-specific/prevent_bad_version_numbers.py @@ -0,0 +1,119 @@ +"""Pre-receive hook to check legality of version bumps. + +This version check follows the guidelines of the Bioconductor +project. The guidelines are given at this link, +http://bioconductor.org/developers/how-to/version-numbering/. +""" + +#!/usr/bin/env python + +from __future__ import print_function +import subprocess +import sys + +ZERO_COMMIT = "0000000000000000000000000000000000000000" + + +def eprint(*args, **kwargs): + """Helper function to print to std err.""" + print(*args, file=sys.stderr, **kwargs) + + +def throw_error(prev_version, new_version): + """Throw error message for every version bump failure.""" + message = ("Error: Illegal version bump from '%s' to '%s'. Check \n" + "http://bioconductor.org/developers/how-to/version-numbering/ \n" + "for details" % (prev_version, new_version)) + sys.exit(message) + return + +def git_diff(oldrev, newrev, fname): + """Git diff between two commits.""" + diff = subprocess.check_output(["git", + "diff", + oldrev + ".." + newrev, + "--", fname]) + return diff.splitlines() + + +def git_diff_pre_commit(fname): + """Git diff for a pre-commit hook.""" + diff = subprocess.check_output(["git", + "diff", + "--cached", fname]) + return diff.splitlines() + + +def git_diff_files(commit): + """Get list of files in diff.""" + files_modified = subprocess.check_output(["git", + "diff-index", + "--name-only", + "--cached", + commit]) + return files_modified.splitlines() + + +def get_version_bump(diff): + """Get the version bumps in DESCRIPTION file.""" + prev_version = [line.replace("-Version:", "") + for line in diff + if line.startswith("-Version")] + new_version = [line.replace("+Version:", "") + for line in diff + if line.startswith("+Version")] + if prev_version == new_version: + return None, None + return prev_version[0].strip(), new_version[0].strip() + + +def check_version_bump(prev_version, new_version, refname): + """Check the version bump for legality.""" + x0, y0, z0 = map(int, prev_version.split(".")) + x, y, z = map(int, new_version.split(".")) + if "RELEASE" in refname: + # x should never change + if x != x0: + throw_error(prev_version, new_version) + # y should be even + if y % 2 != 0: + # y should not be 99 i.e no major version change + if y == 99: + throw_error(prev_version, new_version) + throw_error(prev_version, new_version) + # z should be incremented + if not z - z0 >= 0: + throw_error(prev_version, new_version) + + if "master" in refname: + # x should never change + if x != x0: + throw_error(prev_version, new_version) + # y should be odd + if y % 2 == 0: + throw_error(prev_version, new_version) + # y should not be the same, and can be 99 + if (y != y0) and (y != 99): + throw_error(prev_version, new_version) + # z should be incremented and cannot be 99 + # to indicate major version change + if not (z - z0 >= 0) and (y != 99): + throw_error(prev_version, new_version) + return + + +def prevent_bad_version_numbers(oldrev, newrev, refname): + """Prevent bad version numbers in DESCRIPTION file. + + This function acts as the wrapper for all the helper functions. + """ + files_modified = git_diff_files(newrev) + for fname in files_modified: + if "DESCRIPTION" in fname: + diff = git_diff(oldrev, newrev, fname) + # eprint("DIFF: \n", "\n".join(diff)) + prev_version, new_version = get_version_bump(diff) + if (prev_version is None) and (new_version is None): + continue + check_version_bump(prev_version, new_version, refname) + return diff --git a/hooks/repo-specific/test_prevent_bad_version_numbers.py b/hooks/repo-specific/test_prevent_bad_version_numbers.py new file mode 100644 index 0000000..5cc654e --- /dev/null +++ b/hooks/repo-specific/test_prevent_bad_version_numbers.py @@ -0,0 +1,113 @@ +"""Tests for the pre-receive hook to check version numbers.""" + +import subprocess +import re +import os +import pytest +from hooks.prevent_bad_version_numbers import check_version_bump + + +CWD = "/Users/ni41435_ca/Documents/bioc_git_transition/hooks/repo-specific/test_proj" +DESC = "DESCRIPTION" + + +def change_version(new_version, cwd=CWD): + filename = DESC + path = os.path.join(cwd, filename) + s = open(path).read() + x = re.sub(r"Version: .+\n", "Version: " + new_version + "\n", s) + f = open(path, 'w') + f.write(x) + f.close() + return + +def git_add(path, cwd=CWD): + cmd = ['git', 'add', path] + subprocess.check_call(cmd, cwd=cwd) + return + +def git_checkout(branch, cwd=CWD): + cmd = ['git', 'checkout', branch] + subprocess.check_call(cmd, cwd=cwd) + return + +def git_commit(message, cwd=CWD): + cmd = ['git', 'commit', '-m', message] + subprocess.check_call(cmd, cwd=cwd) + return + +def git_push(cwd=CWD): + cmd = ['git', 'push'] + out = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + return out + +def test_master_check_version_bump(): + # Master + refname = "master" + # y should be odd + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("0.25.1", "0.26.1", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 1 + + # x should not change + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("0.25.1", "1.25.1", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 1 + + # z should change by increment only + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("0.25.5", "0.25.4", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 1 + + # z can be 99 + res = check_version_bump("0.25.4", "0.99.0", refname) + assert res == None + + return + + +def test_release_check_version_bump(): + refname = "RELEASE_3_6" + # y should be even + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("0.26.1", "0.27.1", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 1 + # x should not change + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("0.26.1", "1.26.1", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 1 + # x should not change, even if y changes, it should + # throw the same error. + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("0.25.1", "1.25.1", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 1 + # z should not decrement + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("0.26.4", "0.25.3", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 1 + + # z can be 99 + with pytest.raises(SystemExit) as pytest_wrapped_e: + res = check_version_bump("0.26.4", "0.99.0", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 1 + return + +def test_integration_version_bump_in_master(): + refname = "master" + change_version("1.25.61") + git_add(DESC) + with pytest.raises(SystemExit) as pytest_wrapped_e: + change_version("2.25.61") + git_add(DESC) + git_commit("Fail test", cwd=CWD) + out = git_push() + assert "error" in out + assert pytest_wrapped_e.value.code == 1 From 3505960fdc536d3a41f6c637c408ab792c4d6884 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Fri, 30 Mar 2018 13:49:11 -0400 Subject: [PATCH 35/70] Modifications to pre-receive hook to include prevent_bad_version_numbers --- hooks/repo-specific/pre-receive-hook | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/hooks/repo-specific/pre-receive-hook b/hooks/repo-specific/pre-receive-hook index 4592206..9444cb6 100755 --- a/hooks/repo-specific/pre-receive-hook +++ b/hooks/repo-specific/pre-receive-hook @@ -5,10 +5,12 @@ import sys import fileinput from prevent_large_files import prevent_large_files from prevent_duplicate_commits import prevent_duplicate_commits +from prevent_bad_version_numbers import prevent_bad_version_numbers ZERO_COMMIT = "0000000000000000000000000000000000000000" + # This code doesn't run; enable for local testing. # to use: # @@ -18,7 +20,7 @@ ZERO_COMMIT = "0000000000000000000000000000000000000000" # 4. run python prevent-duplicate-commits from repository root # 5. create branch and repeat for each test if False: - refname=None + refname = None revs = subprocess.check_output([ "git", "log", "-2", "--format=%H" ]).splitlines() @@ -28,7 +30,11 @@ if False: prevent_duplicate_commits(oldrev, newrev, refname) sys.exit(0) + +# This tests the pre-receive hook if __name__ == "__main__": + newestrev = ZERO_COMMIT + oldestrev = ZERO_COMMIT for line in fileinput.input(): std_input = line.split(" ") oldrev, newrev, refname = [elt.strip() for elt in std_input] @@ -39,3 +45,11 @@ if __name__ == "__main__": prevent_large_files(oldrev, newrev, refname) # prevent duplicate commits prevent_duplicate_commits(oldrev, newrev, refname) + # prevent bad version numbers + if newestrev == ZERO_COMMIT: + newestrev = newrev + oldestrev = oldrev + # TODO: REMOVE THIS PRINT STATEMENT + print(newestrev, oldestrev) + if newestrev != ZERO_COMMIT: + prevent_bad_version_numbers(oldestrev, newestrev, refname) From 9790975401a81bcc65462c7c1e220440a1ea5ff5 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Fri, 30 Mar 2018 16:12:09 -0400 Subject: [PATCH 36/70] Add more test cases to check version numbers. -- Refractor code in prevent_bad_version_numbers --- .../prevent_bad_version_numbers.py | 89 ++++++--- .../test_prevent_bad_version_numbers.py | 169 ++++++++++++++++-- 2 files changed, 210 insertions(+), 48 deletions(-) diff --git a/hooks/repo-specific/prevent_bad_version_numbers.py b/hooks/repo-specific/prevent_bad_version_numbers.py index f675918..17cbba6 100644 --- a/hooks/repo-specific/prevent_bad_version_numbers.py +++ b/hooks/repo-specific/prevent_bad_version_numbers.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python """Pre-receive hook to check legality of version bumps. This version check follows the guidelines of the Bioconductor @@ -5,11 +6,11 @@ http://bioconductor.org/developers/how-to/version-numbering/. """ -#!/usr/bin/env python - from __future__ import print_function import subprocess import sys +import re + ZERO_COMMIT = "0000000000000000000000000000000000000000" @@ -27,6 +28,7 @@ def throw_error(prev_version, new_version): sys.exit(message) return + def git_diff(oldrev, newrev, fname): """Git diff between two commits.""" diff = subprocess.check_output(["git", @@ -67,40 +69,69 @@ def get_version_bump(diff): return prev_version[0].strip(), new_version[0].strip() -def check_version_bump(prev_version, new_version, refname): - """Check the version bump for legality.""" +def check_version_format(prev_version, new_version): + """Check format of version.""" + regex = re.compile('\d{1}\.\d{1,2}\.\d{1,3}$') + if not regex.match(new_version): + throw_error(prev_version, new_version) + try: + x0, y0, z0 = map(int, prev_version.split(".")) + x, y, z = map(int, new_version.split(".")) + except ValueError as e: + print('format of version number is wrong') + throw_error(prev_version, new_version) + return prev_version, new_version + + +def check_version_in_release(prev_version, new_version): + """Check version in RELEASE_branch.""" x0, y0, z0 = map(int, prev_version.split(".")) x, y, z = map(int, new_version.split(".")) - if "RELEASE" in refname: - # x should never change - if x != x0: - throw_error(prev_version, new_version) + # x should never change + if x != x0: + throw_error(prev_version, new_version) # y should be even - if y % 2 != 0: - # y should not be 99 i.e no major version change - if y == 99: - throw_error(prev_version, new_version) - throw_error(prev_version, new_version) - # z should be incremented - if not z - z0 >= 0: + if y % 2 != 0: + # y should not be 99 i.e no major version change + if y == 99: throw_error(prev_version, new_version) + throw_error(prev_version, new_version) + # z should be incremented + if not z - z0 >= 0: + throw_error(prev_version, new_version) + return - if "master" in refname: - # x should never change - if x != x0: - throw_error(prev_version, new_version) - # y should be odd - if y % 2 == 0: - throw_error(prev_version, new_version) - # y should not be the same, and can be 99 - if (y != y0) and (y != 99): - throw_error(prev_version, new_version) - # z should be incremented and cannot be 99 - # to indicate major version change - if not (z - z0 >= 0) and (y != 99): - throw_error(prev_version, new_version) + +def check_version_in_master(prev_version, new_version): + """Check version in master branch.""" + x0, y0, z0 = map(int, prev_version.split(".")) + x, y, z = map(int, new_version.split(".")) + # x should never change + if x != x0: + throw_error(prev_version, new_version) + # y should be odd + if y % 2 == 0: + throw_error(prev_version, new_version) + # y should not be the same, and can be 99 + if (y != y0) and (y != 99): + throw_error(prev_version, new_version) + # z should be incremented and cannot be 99 + # to indicate major version change + if not (z - z0 >= 0) and (y != 99): + throw_error(prev_version, new_version) return +def check_version_bump(prev_version, new_version, refname): + """Check the version bump for legality.""" + # Check format of version + prev_version, new_version = check_version_format(prev_version, new_version) + if "RELEASE" in refname: + check_version_in_release(prev_version, new_version) + + if "master" in refname: + check_version_in_master(prev_version, new_version) + return 0 + def prevent_bad_version_numbers(oldrev, newrev, refname): """Prevent bad version numbers in DESCRIPTION file. diff --git a/hooks/repo-specific/test_prevent_bad_version_numbers.py b/hooks/repo-specific/test_prevent_bad_version_numbers.py index 5cc654e..0469f35 100644 --- a/hooks/repo-specific/test_prevent_bad_version_numbers.py +++ b/hooks/repo-specific/test_prevent_bad_version_numbers.py @@ -48,23 +48,23 @@ def test_master_check_version_bump(): with pytest.raises(SystemExit) as pytest_wrapped_e: check_version_bump("0.25.1", "0.26.1", refname) assert pytest_wrapped_e.type == SystemExit - assert pytest_wrapped_e.value.code == 1 + assert pytest_wrapped_e.value.code != 0 # x should not change with pytest.raises(SystemExit) as pytest_wrapped_e: check_version_bump("0.25.1", "1.25.1", refname) assert pytest_wrapped_e.type == SystemExit - assert pytest_wrapped_e.value.code == 1 + assert pytest_wrapped_e.value.code != 0 # z should change by increment only with pytest.raises(SystemExit) as pytest_wrapped_e: check_version_bump("0.25.5", "0.25.4", refname) assert pytest_wrapped_e.type == SystemExit - assert pytest_wrapped_e.value.code == 1 + assert pytest_wrapped_e.value.code != 0 # z can be 99 res = check_version_bump("0.25.4", "0.99.0", refname) - assert res == None + assert res == 0 return @@ -75,39 +75,170 @@ def test_release_check_version_bump(): with pytest.raises(SystemExit) as pytest_wrapped_e: check_version_bump("0.26.1", "0.27.1", refname) assert pytest_wrapped_e.type == SystemExit - assert pytest_wrapped_e.value.code == 1 + assert pytest_wrapped_e.value.code != 0 # x should not change with pytest.raises(SystemExit) as pytest_wrapped_e: check_version_bump("0.26.1", "1.26.1", refname) assert pytest_wrapped_e.type == SystemExit - assert pytest_wrapped_e.value.code == 1 + assert pytest_wrapped_e.value.code != 0 # x should not change, even if y changes, it should # throw the same error. with pytest.raises(SystemExit) as pytest_wrapped_e: check_version_bump("0.25.1", "1.25.1", refname) assert pytest_wrapped_e.type == SystemExit - assert pytest_wrapped_e.value.code == 1 + assert pytest_wrapped_e.value.code != 0 # z should not decrement with pytest.raises(SystemExit) as pytest_wrapped_e: check_version_bump("0.26.4", "0.25.3", refname) assert pytest_wrapped_e.type == SystemExit - assert pytest_wrapped_e.value.code == 1 + assert pytest_wrapped_e.value.code != 0 # z can be 99 with pytest.raises(SystemExit) as pytest_wrapped_e: res = check_version_bump("0.26.4", "0.99.0", refname) assert pytest_wrapped_e.type == SystemExit - assert pytest_wrapped_e.value.code == 1 + assert pytest_wrapped_e.value.code != 0 return -def test_integration_version_bump_in_master(): + +def test_version_bumps_martin(): + refname = "RELEASE_3_6" + ## Tests with bad version number format + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("2.2.2", "2.2.2-1", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code != 0 + + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("2.2.2", "2.2", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code != 0 + + + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("2.2.2", "2.2.2.2", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code != 0 + + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("2.2.2", "2.2-1.2-1", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code != 0 + + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("2.2.2", "2.2.a", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code != 0 + + ## x0 != x1 + + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("2.2.2", "1.2.2", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code != 0 + + res = check_version_bump("2.2.2", "2.2.2", refname) + assert res == 0 + + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("2.2.2", "0.2.2", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code != 0 + return + + +def test_release_version_bumps_martin(): + refname = "RELEASE_3_6" + + res = check_version_bump("2.2.2", "2.2.2", refname) + assert res == 0 + + res = check_version_bump("2.2.2", "2.2.3", refname) + assert res == 0 + + res = check_version_bump("2.2.2", "2.2.10", refname) + assert res == 0 + + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("2.2.2", "2.1.2", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code != 0 + + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("2.2.2", "2.3.2", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code != 0 + + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("2.2.2", "2.99.0", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code != 0 + + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("2.3.2", "2.3.2", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code != 0 + + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("2.3.2", "2.3.3", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code != 0 + return + + +def test_devel_version_bumps_martin(): refname = "master" - change_version("1.25.61") - git_add(DESC) - with pytest.raises(SystemExit) as pytest_wrapped_e: - change_version("2.25.61") - git_add(DESC) - git_commit("Fail test", cwd=CWD) - out = git_push() - assert "error" in out - assert pytest_wrapped_e.value.code == 1 + + res = check_version_bump("2.3.2", "2.3.2", refname) + assert res == 0 + + res = check_version_bump("2.3.2", "2.3.3", refname) + assert res == 0 + + res = check_version_bump("2.3.2", "2.3.10", refname) + assert res == 0 + + res = check_version_bump("2.3.2", "2.99.0", refname) + assert res == 0 + + res = check_version_bump("2.3.2", "2.99.2", refname) + assert res == 0 + + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("2.3.2", "2.4.0", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code != 0 + + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("2.3.2", "2.4.2", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code != 0 + + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("2.3.2", "2.2.2", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code != 0 + + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("2.3.2", "2.3.1", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code != 0 + + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("2.3.2", "2.3.0", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code != 0 + return + + +# def test_integration_version_bump_in_master(): +# refname = "master" +# change_version("1.25.61") +# git_add(DESC) +# with pytest.raises(SystemExit) as pytest_wrapped_e: +# change_version("2.25.61") +# git_add(DESC) +# git_commit("Fail test", cwd=CWD) +# out = git_push() +# assert "error" in out +# assert pytest_wrapped_e.value.code == 1 From 37daff06340b8dacd06bcc6993c68ff25ff56e32 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Wed, 9 May 2018 14:00:39 -0400 Subject: [PATCH 37/70] Changes to prevent_bad_version_numbers - Allow new packages to be pushed - Test new packages to be pushed by making integration test - remove some print statements and add some --- hooks/repo-specific/pre-receive-hook | 2 - hooks/repo-specific/prepare_test.sh | 96 +++++++++++++++++++ .../prevent_bad_version_numbers.py | 25 +++-- 3 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 hooks/repo-specific/prepare_test.sh diff --git a/hooks/repo-specific/pre-receive-hook b/hooks/repo-specific/pre-receive-hook index 9444cb6..8a056bc 100755 --- a/hooks/repo-specific/pre-receive-hook +++ b/hooks/repo-specific/pre-receive-hook @@ -49,7 +49,5 @@ if __name__ == "__main__": if newestrev == ZERO_COMMIT: newestrev = newrev oldestrev = oldrev - # TODO: REMOVE THIS PRINT STATEMENT - print(newestrev, oldestrev) if newestrev != ZERO_COMMIT: prevent_bad_version_numbers(oldestrev, newestrev, refname) diff --git a/hooks/repo-specific/prepare_test.sh b/hooks/repo-specific/prepare_test.sh new file mode 100644 index 0000000..b6dac03 --- /dev/null +++ b/hooks/repo-specific/prepare_test.sh @@ -0,0 +1,96 @@ +# INTEGRATION TEST + + +test_repo() { + ## Create bare repo + gittestpath=/tmp/test_bad_version_numbers.git + hooks=/Users/ni41435_ca/Documents/bioc_git_transition/hooks/repo-specific + gittestrepopath=/tmp/test_bad_version_numbers + + ## Clean up + if [ -d "$gittestpath" ]; then + rm -rf $gittestpath + fi + + if [ -d "$gittestrepopath" ]; then + rm -rf $gittestrepopath + fi + + mkdir $gittestpath + cd $gittestpath + + ## Make bare clone + git init --bare + + ## Copy hooks + cp $hooks/prevent_bad_version_numbers.py $hooks/prevent_duplicate_commits.py $hooks/prevent_large_files.py hooks/ + cp $hooks/pre-receive-hook hooks/pre-receive + + ## Make clone of bare repo + cd /tmp + git clone $gittestpath +} + +## add tests here +################################################################### +## TEST 1: Check the files between multiple commits in the git diff + +## Initiate test repo +test_repo +cd $gittestrepopath + +## 1. Add DESCRIPTION file +cp /tmp/DESCRIPTION . +git add DESCRIPTION +git commit -m "Add DESCRIPTION file" + +## 2. Add dummy file + +touch dummy1 +git add dummy1 +git commit -m "Add dummy1 file" + +## 2. Add dummy file 2 + +touch dummy2 +git add dummy2 +git commit -m "Add dummy2 file" + +## Git push to test + +git push + +################################################################### + +## Test 2: Check bad version bumps + +## Initiate test repo +test_repo +cd $gittestrepopath + +## 1. Add dummy file + +touch dummy1 +git add dummy1 +git commit -m "Add dummy1 file" + +## 2. Add DESCRIPTION file +cp /tmp/DESCRIPTION . +git add DESCRIPTION +git commit -m "Add DESCRIPTION file" + +## 3. Add dummy file 2 + +touch dummy2 +git add dummy2 +git commit -m "Add dummy2 file" + +## 4. Add dummy file 3 + +touch dummy3 +git add dummy3 +git commit -m "Add dummy2 file" + +## Git push to test + +git push diff --git a/hooks/repo-specific/prevent_bad_version_numbers.py b/hooks/repo-specific/prevent_bad_version_numbers.py index 17cbba6..da19763 100644 --- a/hooks/repo-specific/prevent_bad_version_numbers.py +++ b/hooks/repo-specific/prevent_bad_version_numbers.py @@ -46,13 +46,12 @@ def git_diff_pre_commit(fname): return diff.splitlines() -def git_diff_files(commit): +def git_diff_files(oldrev, newrev): """Get list of files in diff.""" files_modified = subprocess.check_output(["git", - "diff-index", + "diff", "--name-only", - "--cached", - commit]) + oldrev + ".." + newrev]) return files_modified.splitlines() @@ -61,11 +60,17 @@ def get_version_bump(diff): prev_version = [line.replace("-Version:", "") for line in diff if line.startswith("-Version")] + eprint("prev", prev_version) new_version = [line.replace("+Version:", "") for line in diff if line.startswith("+Version")] + eprint("new", new_version) + ## If versions are equal, no version change if prev_version == new_version: return None, None + ## No change in DESCRIPTION file from new package push + if not prev_version or not new_version: + return None, None return prev_version[0].strip(), new_version[0].strip() @@ -78,7 +83,7 @@ def check_version_format(prev_version, new_version): x0, y0, z0 = map(int, prev_version.split(".")) x, y, z = map(int, new_version.split(".")) except ValueError as e: - print('format of version number is wrong') + print('format of version number is wrong', e) throw_error(prev_version, new_version) return prev_version, new_version @@ -138,11 +143,17 @@ def prevent_bad_version_numbers(oldrev, newrev, refname): This function acts as the wrapper for all the helper functions. """ - files_modified = git_diff_files(newrev) + if oldrev == ZERO_COMMIT: + if refname == "refs/heads/master": + oldrev = subprocess.check_output([ + "git", "rev-list", "--max-parents=0", newrev + ]).split().pop().strip() + else: + oldrev = "HEAD" + files_modified = git_diff_files(oldrev, newrev) for fname in files_modified: if "DESCRIPTION" in fname: diff = git_diff(oldrev, newrev, fname) - # eprint("DIFF: \n", "\n".join(diff)) prev_version, new_version = get_version_bump(diff) if (prev_version is None) and (new_version is None): continue From fe8e06544d171b048954d67528431f5cd494f95f Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Wed, 9 May 2018 14:05:27 -0400 Subject: [PATCH 38/70] Remove dead code --- hooks/repo-specific/prepare_test.sh | 12 ++++++------ hooks/repo-specific/prevent_bad_version_numbers.py | 9 +++------ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/hooks/repo-specific/prepare_test.sh b/hooks/repo-specific/prepare_test.sh index b6dac03..9c1731c 100644 --- a/hooks/repo-specific/prepare_test.sh +++ b/hooks/repo-specific/prepare_test.sh @@ -6,26 +6,26 @@ test_repo() { gittestpath=/tmp/test_bad_version_numbers.git hooks=/Users/ni41435_ca/Documents/bioc_git_transition/hooks/repo-specific gittestrepopath=/tmp/test_bad_version_numbers - + ## Clean up if [ -d "$gittestpath" ]; then rm -rf $gittestpath fi - + if [ -d "$gittestrepopath" ]; then rm -rf $gittestrepopath fi - + mkdir $gittestpath cd $gittestpath - + ## Make bare clone git init --bare - + ## Copy hooks cp $hooks/prevent_bad_version_numbers.py $hooks/prevent_duplicate_commits.py $hooks/prevent_large_files.py hooks/ cp $hooks/pre-receive-hook hooks/pre-receive - + ## Make clone of bare repo cd /tmp git clone $gittestpath diff --git a/hooks/repo-specific/prevent_bad_version_numbers.py b/hooks/repo-specific/prevent_bad_version_numbers.py index da19763..84394a3 100644 --- a/hooks/repo-specific/prevent_bad_version_numbers.py +++ b/hooks/repo-specific/prevent_bad_version_numbers.py @@ -144,12 +144,9 @@ def prevent_bad_version_numbers(oldrev, newrev, refname): This function acts as the wrapper for all the helper functions. """ if oldrev == ZERO_COMMIT: - if refname == "refs/heads/master": - oldrev = subprocess.check_output([ - "git", "rev-list", "--max-parents=0", newrev - ]).split().pop().strip() - else: - oldrev = "HEAD" + oldrev = subprocess.check_output([ + "git", "rev-list", "--max-parents=0", newrev + ]).split().pop().strip() files_modified = git_diff_files(oldrev, newrev) for fname in files_modified: if "DESCRIPTION" in fname: From d4f24972f7160e670d8f8e700d99ae2915bea096 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Wed, 9 May 2018 14:14:52 -0400 Subject: [PATCH 39/70] Script to allow users to detect duplicate commits Usage: python detect_duplicate_commits.py /tmp/BiocGenerics 100 python detect_duplicate_commits.py --- misc/detect_duplicate_commits.py | 90 ++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 misc/detect_duplicate_commits.py diff --git a/misc/detect_duplicate_commits.py b/misc/detect_duplicate_commits.py new file mode 100644 index 0000000..3cf2b06 --- /dev/null +++ b/misc/detect_duplicate_commits.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python + +import subprocess +import sys +import re +import os + +# Global variables used by pre-recieve hook + + +SVN_COMMIT_REGEX = re.compile(".*git-svn-id: .*@([0-9]{6})") +ZERO_COMMIT = "0000000000000000000000000000000000000000" +ERROR_DUPLICATE_COMMITS = """Error: duplicate commits. + +There are duplicate commits in your commit history, These cannot be +pushed to the Bioconductor git server. Please make sure that this is +resolved. + +Take a look at the documentation to fix this, +https://bioconductor.org/developers/how-to/git/sync-existing-repositories/, +particularly, point #8 (force Bioconductor master to Github master). + +For more information, or help resolving this issue, contact +. Provide the error, the package name and +any other details we might need. + +Use + + git show %s + git show %s + +to see body of commits. +""" + +def get_svn_revision(commit): + body = subprocess.check_output([ "git", "show", "--format=%b", commit ]) + revision = SVN_COMMIT_REGEX.match(body) + if revision != None: + revision = revision.group(1) + return revision + + +def prevent_duplicate_commits(newrev): + """Pre-receive hook to check for duplicate SVN commits.""" + try: + commit_list = subprocess.check_output([ + "git", "rev-list", newrev, "-n", GIT_COMMIT_LIST_LENGTH + ]) + except Exception as e: + print("Exception: %s" % e) + pass + commit_list = commit_list.split("\n") + commit_list = [item for item in commit_list if len(item)>0] + + # For each of the first GIT_COMMIT_LIST_LENGTH pairs, check diff + for i in xrange(len(commit_list) - 1): + first = commit_list[i] + second = commit_list[i+1] + + rev1 = get_svn_revision(first) + rev2 = get_svn_revision(second) + if rev1 and (rev1 == rev2): + diff = subprocess.check_output(["git", "diff", first, second]) + # If the diff of two commits is empty, means they are the same. + # i.e duplicate + if not diff: + print(ERROR_DUPLICATE_COMMITS % (first, second)) + sys.exit(1) + return + + +if __name__ == "__main__": + print("""Usage: + python detect_duplicate_commits.py + + example: + + 'python detect_duplicate_commits.py /mypath/BiocGenerics 100' + + NOTE: this script will stop at the first instance of a duplicate commit. + """) + package_path = sys.argv[1] + GIT_COMMIT_LIST_LENGTH = sys.argv[2] + os.chdir(package_path) + + revs = subprocess.check_output([ + "git", "log", "-2", "--format=%H" + ]).splitlines() + newrev = revs[0].strip() + prevent_duplicate_commits(newrev) From 2d6922a532f721300bf442bd6af5402e9d08a3b2 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Thu, 10 May 2018 10:21:42 -0400 Subject: [PATCH 40/70] Change the oldrev to 4b825d.. Setting oldrev to this commit, will solve the issue, for the ZERO_COMMIT, and new package push. -- 4b825dc642cb6eb9a060e54bf8d69288fbee4904 is the id of the "empty tree" in Git and it's always available in every repository. --- hooks/repo-specific/prevent_bad_version_numbers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hooks/repo-specific/prevent_bad_version_numbers.py b/hooks/repo-specific/prevent_bad_version_numbers.py index 84394a3..2c48de2 100644 --- a/hooks/repo-specific/prevent_bad_version_numbers.py +++ b/hooks/repo-specific/prevent_bad_version_numbers.py @@ -60,11 +60,9 @@ def get_version_bump(diff): prev_version = [line.replace("-Version:", "") for line in diff if line.startswith("-Version")] - eprint("prev", prev_version) new_version = [line.replace("+Version:", "") for line in diff if line.startswith("+Version")] - eprint("new", new_version) ## If versions are equal, no version change if prev_version == new_version: return None, None @@ -144,9 +142,11 @@ def prevent_bad_version_numbers(oldrev, newrev, refname): This function acts as the wrapper for all the helper functions. """ if oldrev == ZERO_COMMIT: - oldrev = subprocess.check_output([ - "git", "rev-list", "--max-parents=0", newrev - ]).split().pop().strip() + ## https://stackoverflow.com/questions/40883798/how-to-get-git-diff-of-the-first-commit + ## 4b825dc642cb6eb9a060e54bf8d69288fbee4904 is the + ## id of the "empty tree" in Git and it's always + ## available in every repository. + oldrev = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" files_modified = git_diff_files(oldrev, newrev) for fname in files_modified: if "DESCRIPTION" in fname: From 4145aef48f997f797fa9a79832347ce209879112 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Thu, 10 May 2018 10:29:22 -0400 Subject: [PATCH 41/70] Make the hooks testable within the test. add __init__.py to make the python code importable --- hooks/repo-specific/prepare_test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hooks/repo-specific/prepare_test.sh b/hooks/repo-specific/prepare_test.sh index 9c1731c..7f2b047 100644 --- a/hooks/repo-specific/prepare_test.sh +++ b/hooks/repo-specific/prepare_test.sh @@ -26,6 +26,8 @@ test_repo() { cp $hooks/prevent_bad_version_numbers.py $hooks/prevent_duplicate_commits.py $hooks/prevent_large_files.py hooks/ cp $hooks/pre-receive-hook hooks/pre-receive + cp $hooks/test_prevent_bad_version_numbers.py hooks/ + touch hooks/__init__.py ## Make clone of bare repo cd /tmp git clone $gittestpath From def36e5ff299a594c8113432397407fcd25562b5 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Fri, 11 May 2018 10:54:07 -0400 Subject: [PATCH 42/70] Script to add missing packages in packages.conf This script checks the manifest files for missing packages in packages.conf. Adds them to the packages.conf file. --- misc/script.R | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 misc/script.R diff --git a/misc/script.R b/misc/script.R new file mode 100644 index 0000000..93db983 --- /dev/null +++ b/misc/script.R @@ -0,0 +1,63 @@ +library(tidyverse) + +get_bioc_manifest <- + function(manifest) +{ + args <- c("archive", + "--remote=git@git.bioconductor.org:admin/manifest", "HEAD", + manifest,"|" , "tar", "-x") + system2("git", args, wait=TRUE) + f <- readLines(manifest) + f <- sub( + "Package: *", "", + regmatches(f, regexpr("Package:.*", f)) + ) + tbl_df(f) +} + +get_packages_conf <- + function(path = "/Users/ni41435_ca/Documents/gitolite-admin/conf/packages.conf") +{ + software <- readLines(path) + idx <- grep("repo packages/", software) + packs <- gsub("repo packages/", "",software[idx]) + tbl_df(packs) +} + + +## Get manifests +software <- get_bioc_manifest("software.txt") +data_exp <- get_bioc_manifest("data-experiment.txt") +workflows <- get_bioc_manifest("workflows.txt") + +## Add type of package +software$type <- "" +data_exp$type <- "-dataexp-workflow" +workflows$type <- "-dataexp-workflow" + +## bind rows +manifest <- bind_rows(software, data_exp, workflows) + +## get packages conf +packages_conf <- get_packages_conf() + +## packages to add to packages.conf +to_add <- anti_join(manifest, packages_conf) + +## Write packages to temp file called packages.conf +write_file <- + function(tbl) +{ + package_template <- paste0( + "", + "\nrepo packages/%s", + "\n RW master = @none", + "\n RW RELEASE_3_7 = @none", + "\n option hook.pre-receive = pre-receive-hook%s" + ) + template = sprintf(package_template, + tbl$value, + tbl$type) + template = paste(template, collapse="\n") + write(template, file = "packages.conf", append=TRUE) +} From 6e3969d5575122e1d894e9e6548095883fb02225 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Mon, 21 May 2018 11:00:17 -0400 Subject: [PATCH 43/70] Changes to script.R --- misc/script.R | 94 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 3 deletions(-) diff --git a/misc/script.R b/misc/script.R index 93db983..f22edbe 100644 --- a/misc/script.R +++ b/misc/script.R @@ -1,5 +1,16 @@ +library(httr) library(tidyverse) +BIOC_CREDENTIALS_PASSWORD = readLines("~/gitCredentialsAuth.txt") + +auth <- authenticate( + "nitesh.turaga@gmail.com", + BIOC_CREDENTIALS_PASSWORD +) + +.BIOC_CREDENTIALS_URL <- + "https://git.bioconductor.org/BiocCredentials/api/biocusers/" + get_bioc_manifest <- function(manifest) { @@ -31,7 +42,7 @@ data_exp <- get_bioc_manifest("data-experiment.txt") workflows <- get_bioc_manifest("workflows.txt") ## Add type of package -software$type <- "" +software$type <- "-software" data_exp$type <- "-dataexp-workflow" workflows$type <- "-dataexp-workflow" @@ -39,11 +50,13 @@ workflows$type <- "-dataexp-workflow" manifest <- bind_rows(software, data_exp, workflows) ## get packages conf +# packages_conf <- get_packages_conf(path="/tmp/packages.conf") packages_conf <- get_packages_conf() ## packages to add to packages.conf to_add <- anti_join(manifest, packages_conf) + ## Write packages to temp file called packages.conf write_file <- function(tbl) @@ -51,13 +64,88 @@ write_file <- package_template <- paste0( "", "\nrepo packages/%s", - "\n RW master = @none", - "\n RW RELEASE_3_7 = @none", + "\n RW master = %s", + "\n RW RELEASE_3_7 = %s", "\n option hook.pre-receive = pre-receive-hook%s" ) template = sprintf(package_template, tbl$value, + tbl$creds, + tbl$creds, tbl$type) template = paste(template, collapse="\n") write(template, file = "packages.conf", append=TRUE) } + + +get_maintainers_email <- + function(package) +{ + args <- c("archive", + paste0("--remote=git@git.bioconductor.org:packages/", package), + "HEAD", + "DESCRIPTION","|" , "tar", "-x") + system2("git", args, wait=TRUE) + dcf <- read.dcf(file.path("DESCRIPTION")) + if ("Maintainer" %in% colnames(dcf)) { + m <- dcf[, "Maintainer"] + ret <- regexec("<([^>]*)>", m)[[1]] + ml <- attr(ret, "match.length") + email <- substr(m, ret[2], ret[2] + ml[2] - 1) + } + else if ("Authors@R" %in% colnames(dcf)) { + ar <- dcf[, "Authors@R"] + env <- new.env(parent = emptyenv()) + env[["c"]] = c + env[["person"]] <- utils::person + pp <- parse(text = ar, keep.source = TRUE) + tryCatch(people <- eval(pp, env), error = function(e) { + return() + }) + for (person in people) { + if ("cre" %in% person$role) { + email <- person$email + } + } + } + c(package, email) +} + + +get_bioc_credentials <- + function(email_id, auth) +{ + query <- paste0(.BIOC_CREDENTIALS_URL, "query_by_email/%s/") + queries <- sprintf(query, email_id) + content(httr::GET(queries, auth)) +} + + +get_all <- function(package) { + message("package: ", package) + res <- get_maintainers_email(package) + credential <- get_bioc_credentials(res[2], auth=auth) + c(package, credential) +} + + +get_all_safely <- safely(get_all) + +result <- to_add$value %>% map(get_all_safely) + +t_result <- transpose(result)[["result"]] + +## make a tbl with pacakge, credential +creds = tibble(value=unlist(sapply(t_result, `[[`, c(1))), + creds=unlist(sapply(t_result, `[[`, c(2)))) + +## left_join (result_tbl, to_add) using value +to_add <- left_join(to_add, creds, by = c("value")) +to_add$creds <- str_replace_na(to_add$creds, "maintainer") +to_add$creds <- str_replace(to_add$creds, "maintainer", "@bioconductor_writers") +to_add$creds[to_add$creds == ""] <- "@bioconductor_writers" + +write_file(to_add) + +## Packages in the packages.conf, but not in manifest +setdiff( packages_conf$value, manifest$value) From efbb3b7fb598b014176929cb5b884b16dfbed076 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Mon, 21 May 2018 15:55:07 -0400 Subject: [PATCH 44/70] remove script --- misc/script.R | 151 -------------------------------------------------- 1 file changed, 151 deletions(-) delete mode 100644 misc/script.R diff --git a/misc/script.R b/misc/script.R deleted file mode 100644 index f22edbe..0000000 --- a/misc/script.R +++ /dev/null @@ -1,151 +0,0 @@ -library(httr) -library(tidyverse) - -BIOC_CREDENTIALS_PASSWORD = readLines("~/gitCredentialsAuth.txt") - -auth <- authenticate( - "nitesh.turaga@gmail.com", - BIOC_CREDENTIALS_PASSWORD -) - -.BIOC_CREDENTIALS_URL <- - "https://git.bioconductor.org/BiocCredentials/api/biocusers/" - -get_bioc_manifest <- - function(manifest) -{ - args <- c("archive", - "--remote=git@git.bioconductor.org:admin/manifest", "HEAD", - manifest,"|" , "tar", "-x") - system2("git", args, wait=TRUE) - f <- readLines(manifest) - f <- sub( - "Package: *", "", - regmatches(f, regexpr("Package:.*", f)) - ) - tbl_df(f) -} - -get_packages_conf <- - function(path = "/Users/ni41435_ca/Documents/gitolite-admin/conf/packages.conf") -{ - software <- readLines(path) - idx <- grep("repo packages/", software) - packs <- gsub("repo packages/", "",software[idx]) - tbl_df(packs) -} - - -## Get manifests -software <- get_bioc_manifest("software.txt") -data_exp <- get_bioc_manifest("data-experiment.txt") -workflows <- get_bioc_manifest("workflows.txt") - -## Add type of package -software$type <- "-software" -data_exp$type <- "-dataexp-workflow" -workflows$type <- "-dataexp-workflow" - -## bind rows -manifest <- bind_rows(software, data_exp, workflows) - -## get packages conf -# packages_conf <- get_packages_conf(path="/tmp/packages.conf") -packages_conf <- get_packages_conf() - -## packages to add to packages.conf -to_add <- anti_join(manifest, packages_conf) - - -## Write packages to temp file called packages.conf -write_file <- - function(tbl) -{ - package_template <- paste0( - "", - "\nrepo packages/%s", - "\n RW master = %s", - "\n RW RELEASE_3_7 = %s", - "\n option hook.pre-receive = pre-receive-hook%s" - ) - template = sprintf(package_template, - tbl$value, - tbl$creds, - tbl$creds, - tbl$type) - template = paste(template, collapse="\n") - write(template, file = "packages.conf", append=TRUE) -} - - -get_maintainers_email <- - function(package) -{ - args <- c("archive", - paste0("--remote=git@git.bioconductor.org:packages/", package), - "HEAD", - "DESCRIPTION","|" , "tar", "-x") - system2("git", args, wait=TRUE) - dcf <- read.dcf(file.path("DESCRIPTION")) - if ("Maintainer" %in% colnames(dcf)) { - m <- dcf[, "Maintainer"] - ret <- regexec("<([^>]*)>", m)[[1]] - ml <- attr(ret, "match.length") - email <- substr(m, ret[2], ret[2] + ml[2] - 1) - } - else if ("Authors@R" %in% colnames(dcf)) { - ar <- dcf[, "Authors@R"] - env <- new.env(parent = emptyenv()) - env[["c"]] = c - env[["person"]] <- utils::person - pp <- parse(text = ar, keep.source = TRUE) - tryCatch(people <- eval(pp, env), error = function(e) { - return() - }) - for (person in people) { - if ("cre" %in% person$role) { - email <- person$email - } - } - } - c(package, email) -} - - -get_bioc_credentials <- - function(email_id, auth) -{ - query <- paste0(.BIOC_CREDENTIALS_URL, "query_by_email/%s/") - queries <- sprintf(query, email_id) - content(httr::GET(queries, auth)) -} - - -get_all <- function(package) { - message("package: ", package) - res <- get_maintainers_email(package) - credential <- get_bioc_credentials(res[2], auth=auth) - c(package, credential) -} - - -get_all_safely <- safely(get_all) - -result <- to_add$value %>% map(get_all_safely) - -t_result <- transpose(result)[["result"]] - -## make a tbl with pacakge, credential -creds = tibble(value=unlist(sapply(t_result, `[[`, c(1))), - creds=unlist(sapply(t_result, `[[`, c(2)))) - -## left_join (result_tbl, to_add) using value -to_add <- left_join(to_add, creds, by = c("value")) -to_add$creds <- str_replace_na(to_add$creds, "maintainer") -to_add$creds <- str_replace(to_add$creds, "maintainer", "@bioconductor_writers") -to_add$creds[to_add$creds == ""] <- "@bioconductor_writers" - -write_file(to_add) - -## Packages in the packages.conf, but not in manifest -setdiff( packages_conf$value, manifest$value) From 9ef2dcc5c861050491c55f2f9ea9ce428cdd035d Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Mon, 1 Oct 2018 10:24:53 -0400 Subject: [PATCH 45/70] Update .gitignore and don't add 'test' files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index d6cf5ce..954f157 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ admin-docs/*Bioc2017* admin-docs/presentations/.ipynb_checkpoints/ admin-docs/presentations/Demo-bioc2017.ipynb admin-docs/presentations/core-team-transition.ipynb +hooks/repo-specific/test_proj.git/* +hooks/repo-specific/tests/* From b0b6e7e17aedaa8065c1890167368674a75654b2 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Mon, 1 Oct 2018 10:45:31 -0400 Subject: [PATCH 46/70] Add pre-receive hooks for both software and data experiment Minor changes to length of RSS logs --- .../pre-receive-hook-dataexp-workflow | 27 ++++++++++ hooks/repo-specific/pre-receive-hook-software | 50 +++++++++++++++++++ .../prevent_duplicate_commits.py | 2 +- hooks/repo-specific/rss_feed.py | 3 +- 4 files changed, 80 insertions(+), 2 deletions(-) create mode 100755 hooks/repo-specific/pre-receive-hook-dataexp-workflow create mode 100755 hooks/repo-specific/pre-receive-hook-software diff --git a/hooks/repo-specific/pre-receive-hook-dataexp-workflow b/hooks/repo-specific/pre-receive-hook-dataexp-workflow new file mode 100755 index 0000000..2a06a3b --- /dev/null +++ b/hooks/repo-specific/pre-receive-hook-dataexp-workflow @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +import subprocess +import sys +import fileinput +from prevent_duplicate_commits import prevent_duplicate_commits +from prevent_bad_version_numbers import prevent_bad_version_numbers + +ZERO_COMMIT = "0000000000000000000000000000000000000000" + +if __name__ == "__main__": + newestrev = ZERO_COMMIT + oldestrev = ZERO_COMMIT + for line in fileinput.input(): + std_input = line.split(" ") + oldrev, newrev, refname = [elt.strip() for elt in std_input] + # Check for zero commit, check branch deletions + if newrev == ZERO_COMMIT: + continue + # prevent duplicate commits + prevent_duplicate_commits(oldrev, newrev, refname) + # prevent bad version numbers + if newestrev == ZERO_COMMIT: + newestrev = newrev + oldestrev = oldrev + if newestrev != ZERO_COMMIT: + prevent_bad_version_numbers(oldestrev, newestrev, refname) \ No newline at end of file diff --git a/hooks/repo-specific/pre-receive-hook-software b/hooks/repo-specific/pre-receive-hook-software new file mode 100755 index 0000000..be99f07 --- /dev/null +++ b/hooks/repo-specific/pre-receive-hook-software @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +import subprocess +import sys +import fileinput +from prevent_large_files import prevent_large_files +from prevent_duplicate_commits import prevent_duplicate_commits +from prevent_bad_version_numbers import prevent_bad_version_numbers + +ZERO_COMMIT = "0000000000000000000000000000000000000000" + +# This code doesn't run; enable for local testing. +# to use: +# +# 1. Toggle 'False' to 'True' +# 2. make a git repostiory +# 3. create invalid commit history +# 4. run python prevent-duplicate-commits from repository root +# 5. create branch and repeat for each test +if False: + refname=None + revs = subprocess.check_output([ + "git", "log", "-2", "--format=%H" + ]).splitlines() + newrev = revs[0].strip() + oldrev = revs[1].strip() + prevent_large_files(oldrev, newrev, refname) + prevent_duplicate_commits(oldrev, newrev, refname) + sys.exit(0) + + +if __name__ == "__main__": + newestrev = ZERO_COMMIT + oldestrev = ZERO_COMMIT + for line in fileinput.input(): + std_input = line.split(" ") + oldrev, newrev, refname = [elt.strip() for elt in std_input] + # Check for zero commit, check branch deletions + if newrev == ZERO_COMMIT: + continue + # prevent large files + prevent_large_files(oldrev, newrev, refname) + # prevent duplicate commits + prevent_duplicate_commits(oldrev, newrev, refname) + # prevent bad version numbers + if newestrev == ZERO_COMMIT: + newestrev = newrev + oldestrev = oldrev + if newestrev != ZERO_COMMIT: + prevent_bad_version_numbers(oldestrev, newestrev, refname) diff --git a/hooks/repo-specific/prevent_duplicate_commits.py b/hooks/repo-specific/prevent_duplicate_commits.py index 7418f90..3b300b4 100644 --- a/hooks/repo-specific/prevent_duplicate_commits.py +++ b/hooks/repo-specific/prevent_duplicate_commits.py @@ -6,7 +6,7 @@ # Global variables used by pre-recieve hook -GIT_COMMIT_LIST_LENGTH = "50" +GIT_COMMIT_LIST_LENGTH = "30" SVN_COMMIT_REGEX = re.compile(".*git-svn-id: .*@([0-9]{6})") ZERO_COMMIT = "0000000000000000000000000000000000000000" ERROR_DUPLICATE_COMMITS = """Error: duplicate commits. diff --git a/hooks/repo-specific/rss_feed.py b/hooks/repo-specific/rss_feed.py index 27b67c9..f914d09 100644 --- a/hooks/repo-specific/rss_feed.py +++ b/hooks/repo-specific/rss_feed.py @@ -1,4 +1,6 @@ import subprocess +import datetime +# import re from os.path import basename, abspath from xml.etree.ElementTree import fromstring import logging @@ -55,4 +57,3 @@ def rss_feed(oldrev, newrev, refname, length): # Add entry as element in xml.etree entry_list.append(fromstring(entry)) return entry_list - From 2c5b959f77fc8dde6b97020d91dacc442a546ca6 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Mon, 1 Oct 2018 10:47:35 -0400 Subject: [PATCH 47/70] Remove pre-receive hook before it was modularized --- hooks/repo-specific/pre-receive-hook | 53 ---------------------------- 1 file changed, 53 deletions(-) delete mode 100755 hooks/repo-specific/pre-receive-hook diff --git a/hooks/repo-specific/pre-receive-hook b/hooks/repo-specific/pre-receive-hook deleted file mode 100755 index 8a056bc..0000000 --- a/hooks/repo-specific/pre-receive-hook +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python - -import subprocess -import sys -import fileinput -from prevent_large_files import prevent_large_files -from prevent_duplicate_commits import prevent_duplicate_commits -from prevent_bad_version_numbers import prevent_bad_version_numbers - - -ZERO_COMMIT = "0000000000000000000000000000000000000000" - - -# This code doesn't run; enable for local testing. -# to use: -# -# 1. Toggle 'False' to 'True' -# 2. make a git repostiory -# 3. create invalid commit history -# 4. run python prevent-duplicate-commits from repository root -# 5. create branch and repeat for each test -if False: - refname = None - revs = subprocess.check_output([ - "git", "log", "-2", "--format=%H" - ]).splitlines() - newrev = revs[0].strip() - oldrev = revs[1].strip() - prevent_large_files(oldrev, newrev, refname) - prevent_duplicate_commits(oldrev, newrev, refname) - sys.exit(0) - - -# This tests the pre-receive hook -if __name__ == "__main__": - newestrev = ZERO_COMMIT - oldestrev = ZERO_COMMIT - for line in fileinput.input(): - std_input = line.split(" ") - oldrev, newrev, refname = [elt.strip() for elt in std_input] - # Check for zero commit, check branch deletions - if newrev == ZERO_COMMIT: - continue - # prevent large files - prevent_large_files(oldrev, newrev, refname) - # prevent duplicate commits - prevent_duplicate_commits(oldrev, newrev, refname) - # prevent bad version numbers - if newestrev == ZERO_COMMIT: - newestrev = newrev - oldestrev = oldrev - if newestrev != ZERO_COMMIT: - prevent_bad_version_numbers(oldestrev, newestrev, refname) From 888e7c230b7d6d4e6186ab56a64be15d7cbaff5b Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Mon, 1 Oct 2018 10:48:43 -0400 Subject: [PATCH 48/70] Remove disable-large-commits shell script, as it was implemented in python --- hooks/repo-specific/disable-large-commits | 42 ----------------------- 1 file changed, 42 deletions(-) delete mode 100755 hooks/repo-specific/disable-large-commits diff --git a/hooks/repo-specific/disable-large-commits b/hooks/repo-specific/disable-large-commits deleted file mode 100755 index d752dad..0000000 --- a/hooks/repo-specific/disable-large-commits +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -# -# Pre-receive hook that will block any new commits that contain files larger than 5Mb in size. -# - -GITCMD="/usr/bin/git" -zero_commit="0000000000000000000000000000000000000000" -MAXSIZE="5000000" # 5MB limit on file size - -# Read stdin for ref information -while read oldrev newrev refname; do - # Skip branch deletions - if [ "${newrev}" = "${zero_commit}" ]; then - continue; - fi - - # Set oldrev properly if this is branch creation. - if [ "${oldrev}" = "${zero_commit}" ]; then - oldrev="HEAD" - fi - - # Get list of files to look at using git diff - for file in $($GITCMD diff --stat --name-only --diff-filter=ACMRT ${oldrev}..${newrev}); do - # Get the size of this file - size=$($GITCMD cat-file -s ${newrev}:${file}) - # Check to see if for some reason we didn't get a size - if [ ! -z ${size} ]; then - # Compare filesize to MAXSIZE - if [ "${size}" -gt "${MAXSIZE}" ]; then - # Send output back to the user about oversized files. - echo "Error: ${file} larger than 5Mb. Please see Biocondcutor guidelines" - echo " https://bioconductor.org/developers/package-guidelines/" - # Fail here - exit 1 - fi # End size comparison - fi # End check for empty size - done # End list of files -done # End reading stdin - -# If successful, pass -exit 0 From 6c87ec4964a7b308f4e02a0ac2ac52f003119768 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Tue, 2 Oct 2018 10:45:22 -0400 Subject: [PATCH 49/70] README file describing the hooks on the Bioc GIT sever --- hooks/repo-specific/README.md | 58 +++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 hooks/repo-specific/README.md diff --git a/hooks/repo-specific/README.md b/hooks/repo-specific/README.md new file mode 100644 index 0000000..a909fe6 --- /dev/null +++ b/hooks/repo-specific/README.md @@ -0,0 +1,58 @@ +Git hooks for Bioconductor +========================= + +This document describes the hooks on the Bioconductor git server.There +are two types of hooks on the Bioconductor git server, + +1. Pre-receive hooks : These hooks intercept the push from the author + of the package and show an error if their commit does not pass the + "check" the hook performs. + + There are three pre-recieve hooks on the system, + + 1. Prevent Large files: This hook prevents large files from + entering the git repository, where each file can have a max + size of 5MB. + + 1. Prevent bad version numbers: This hook prevents bad version + numbers according to the documentation given in + http://bioconductor.org/developers/how-to/version-numbering/. + + 1. Prevent duplicate commits: This hook checks the last 50 + commits to see if there are any duplicate commits. + +1. Post-receive hooks: This hook takes the commit after it is accepted + into the Bioconductor git server, and processes it for other needs. + + There is currently only one post-receive hook on the system, + + 1. RSS feed: Once a commit is accepted into the system, the + post-receive hook takes the commit information, eg: the + message, the date and the author information, and publishes + it to the GIT log on the Bioconductor website. It also makes + builds the RSS file(xml format) for the feed. + + +The hooks are applied differently to both software and +workflow/data-experiment packages. + +Hooks applied to Software packages: + + * Prevent large files + + * Prevent bad version numbers + + * Prevent duplicate commits + + * RSS feed + +Hooks applied to Workflow/Data-Experiment packages: + + * Prevent bad version numbers + + * Prevent duplicate commits + + * RSS feed + + + From 8aa57af944d3960ef4c724978fed1d642b51e3c7 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Tue, 2 Oct 2018 10:59:49 -0400 Subject: [PATCH 50/70] Minor edits to get markdown formatting --- hooks/repo-specific/README.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/hooks/repo-specific/README.md b/hooks/repo-specific/README.md index a909fe6..b3471b6 100644 --- a/hooks/repo-specific/README.md +++ b/hooks/repo-specific/README.md @@ -10,27 +10,27 @@ are two types of hooks on the Bioconductor git server, There are three pre-recieve hooks on the system, - 1. Prevent Large files: This hook prevents large files from + 1. Prevent Large files: This hook prevents large files from entering the git repository, where each file can have a max size of 5MB. - 1. Prevent bad version numbers: This hook prevents bad version + 1. Prevent bad version numbers: This hook prevents bad version numbers according to the documentation given in http://bioconductor.org/developers/how-to/version-numbering/. - 1. Prevent duplicate commits: This hook checks the last 50 - commits to see if there are any duplicate commits. + 1. Prevent duplicate commits: This hook checks the last 50 commits + to see if there are any duplicate commits. 1. Post-receive hooks: This hook takes the commit after it is accepted into the Bioconductor git server, and processes it for other needs. There is currently only one post-receive hook on the system, - 1. RSS feed: Once a commit is accepted into the system, the + 1. RSS feed: Once a commit is accepted into the system, the post-receive hook takes the commit information, eg: the message, the date and the author information, and publishes - it to the GIT log on the Bioconductor website. It also makes - builds the RSS file(xml format) for the feed. + it to the GIT log on the Bioconductor website. It also makes + builds the RSS file(xml format) for the feed. The hooks are applied differently to both software and @@ -38,21 +38,21 @@ workflow/data-experiment packages. Hooks applied to Software packages: - * Prevent large files +* Prevent large files + +* Prevent bad version numbers - * Prevent bad version numbers + Prevent duplicate commits - * Prevent duplicate commits - - * RSS feed +* RSS feed Hooks applied to Workflow/Data-Experiment packages: - * Prevent bad version numbers - - * Prevent duplicate commits - - * RSS feed +* Prevent bad version numbers + +* Prevent duplicate commits + +* RSS feed From db1113a482bb27263466ed98a8c0f3b4cc993fb5 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Tue, 2 Oct 2018 11:01:17 -0400 Subject: [PATCH 51/70] markdown formatting --- hooks/repo-specific/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/repo-specific/README.md b/hooks/repo-specific/README.md index b3471b6..ac9cbc8 100644 --- a/hooks/repo-specific/README.md +++ b/hooks/repo-specific/README.md @@ -42,7 +42,7 @@ Hooks applied to Software packages: * Prevent bad version numbers - Prevent duplicate commits +* Prevent duplicate commits * RSS feed From 831128b896aee8f6cf7eb8cb30a77be3af750245 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Thu, 4 Oct 2018 13:11:14 -0400 Subject: [PATCH 52/70] This commit adds more modular control to the hooks --- hooks/repo-specific/pre-receive-hook-software | 59 +++++++++++++++++-- 1 file changed, 53 insertions(+), 6 deletions(-) diff --git a/hooks/repo-specific/pre-receive-hook-software b/hooks/repo-specific/pre-receive-hook-software index be99f07..cd21673 100755 --- a/hooks/repo-specific/pre-receive-hook-software +++ b/hooks/repo-specific/pre-receive-hook-software @@ -2,12 +2,16 @@ import subprocess import sys +import os +import re import fileinput from prevent_large_files import prevent_large_files from prevent_duplicate_commits import prevent_duplicate_commits from prevent_bad_version_numbers import prevent_bad_version_numbers ZERO_COMMIT = "0000000000000000000000000000000000000000" +HOOKS_CONF = "file:///home/git/repositories/admin/hook_maintainer.git" +LOCAL_HOOKS_CONF = "file:////Users/ni41435_ca/Documents/hook_maintainer.git" # This code doesn't run; enable for local testing. # to use: @@ -28,23 +32,66 @@ if False: prevent_duplicate_commits(oldrev, newrev, refname) sys.exit(0) - -if __name__ == "__main__": + +# TODO: Change LOCAL_HOOKS_CONF to HOOKS_CONF +def get_hooks_conf(): + cmd = "git archive --remote=" + LOCAL_HOOKS_CONF + " HEAD hooks.conf | tar -x" + res = subprocess.check_output(cmd, shell=True, cwd="/tmp") + if os.path.exists("/tmp/hooks.conf"): + with open("/tmp/hooks.conf") as f: + conf = f.read() + return conf + + +def read_bioc_conf(conf = get_hooks_conf()): + d = {} + res = [pack.strip().split("\n") for pack in conf.split("\n\n")] + for item in res: + d[item[0]] = item[1:] + # Get package name, this works because the script is run inside the package + package_name = re.search('/.*/(.+?.git).*', + os.getcwd()).group(1).replace(".git", "") + package = "Package: " + package_name + large_files = True + version_numbers = True + duplicate_commits = True + if "pre-receive-hook-large-files: False" in d[package]: + large_files = False + if "pre-receive-hook-version-numbers: False" in d[package]: + version_numbers = False + if "pre-receive-hook-duplicate-commits: False" in d[package]: + duplicate_commits = False + return (large_files, version_numbers, duplicate_commits) + + +def apply_hooks(large_files, version_numbers, duplicate_commits): newestrev = ZERO_COMMIT oldestrev = ZERO_COMMIT for line in fileinput.input(): std_input = line.split(" ") + print(std_input) oldrev, newrev, refname = [elt.strip() for elt in std_input] # Check for zero commit, check branch deletions if newrev == ZERO_COMMIT: continue # prevent large files - prevent_large_files(oldrev, newrev, refname) + if large_files: # enable hook + print("large_files: ", large_files) + prevent_large_files(oldrev, newrev, refname) # prevent duplicate commits - prevent_duplicate_commits(oldrev, newrev, refname) - # prevent bad version numbers + if duplicate_commits: # enable hook + print("duplicate commit: ", duplicate_commits) + prevent_duplicate_commits(oldrev, newrev, refname) if newestrev == ZERO_COMMIT: newestrev = newrev oldestrev = oldrev - if newestrev != ZERO_COMMIT: + # prevent bad version numbers + print("version_numbers and newestrev != ZERO_COMMIT", version_numbers and newestrev != ZERO_COMMIT) + if (version_numbers and newestrev != ZERO_COMMIT): # enable hook + print("version_numbers: ", version_numbers) prevent_bad_version_numbers(oldestrev, newestrev, refname) + + +if __name__ == "__main__": + large_files, version_numbers, duplicate_commits = read_bioc_conf() + apply_hooks(large_files, version_numbers, duplicate_commits) From 82ab77579b226fde8b93f12dad7d4b1de0cee465 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Fri, 5 Oct 2018 11:52:09 -0400 Subject: [PATCH 53/70] Update prepare-test and pre-receive-hook-software integration test --- hooks/repo-specific/prepare_test.sh | 2 +- .../test_prevent_bad_version_numbers.py | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/hooks/repo-specific/prepare_test.sh b/hooks/repo-specific/prepare_test.sh index 7f2b047..e032d60 100644 --- a/hooks/repo-specific/prepare_test.sh +++ b/hooks/repo-specific/prepare_test.sh @@ -24,7 +24,7 @@ test_repo() { ## Copy hooks cp $hooks/prevent_bad_version_numbers.py $hooks/prevent_duplicate_commits.py $hooks/prevent_large_files.py hooks/ - cp $hooks/pre-receive-hook hooks/pre-receive + cp $hooks/pre-receive-hook-software hooks/pre-receive cp $hooks/test_prevent_bad_version_numbers.py hooks/ touch hooks/__init__.py diff --git a/hooks/repo-specific/test_prevent_bad_version_numbers.py b/hooks/repo-specific/test_prevent_bad_version_numbers.py index 0469f35..3ef28db 100644 --- a/hooks/repo-specific/test_prevent_bad_version_numbers.py +++ b/hooks/repo-specific/test_prevent_bad_version_numbers.py @@ -6,7 +6,6 @@ import pytest from hooks.prevent_bad_version_numbers import check_version_bump - CWD = "/Users/ni41435_ca/Documents/bioc_git_transition/hooks/repo-specific/test_proj" DESC = "DESCRIPTION" @@ -21,26 +20,31 @@ def change_version(new_version, cwd=CWD): f.close() return + def git_add(path, cwd=CWD): cmd = ['git', 'add', path] subprocess.check_call(cmd, cwd=cwd) return + def git_checkout(branch, cwd=CWD): cmd = ['git', 'checkout', branch] subprocess.check_call(cmd, cwd=cwd) return + def git_commit(message, cwd=CWD): cmd = ['git', 'commit', '-m', message] subprocess.check_call(cmd, cwd=cwd) return + def git_push(cwd=CWD): cmd = ['git', 'push'] out = subprocess.check_output(cmd, stderr=subprocess.STDOUT) return out + def test_master_check_version_bump(): # Master refname = "master" @@ -103,7 +107,7 @@ def test_release_check_version_bump(): def test_version_bumps_martin(): refname = "RELEASE_3_6" - ## Tests with bad version number format + # Tests with bad version number format with pytest.raises(SystemExit) as pytest_wrapped_e: check_version_bump("2.2.2", "2.2.2-1", refname) assert pytest_wrapped_e.type == SystemExit @@ -114,7 +118,6 @@ def test_version_bumps_martin(): assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code != 0 - with pytest.raises(SystemExit) as pytest_wrapped_e: check_version_bump("2.2.2", "2.2.2.2", refname) assert pytest_wrapped_e.type == SystemExit @@ -130,8 +133,7 @@ def test_version_bumps_martin(): assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code != 0 - ## x0 != x1 - + # x0 != x1 with pytest.raises(SystemExit) as pytest_wrapped_e: check_version_bump("2.2.2", "1.2.2", refname) assert pytest_wrapped_e.type == SystemExit From bfc52eff66b19a231a7403d00de2d08790351ace Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Fri, 5 Oct 2018 12:00:01 -0400 Subject: [PATCH 54/70] Modify .gitignore and add test repos --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 954f157..63af3c3 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,6 @@ admin-docs/*Bioc2017* admin-docs/presentations/.ipynb_checkpoints/ admin-docs/presentations/Demo-bioc2017.ipynb admin-docs/presentations/core-team-transition.ipynb -hooks/repo-specific/test_proj.git/* +hooks/repo-specific/test_proj* hooks/repo-specific/tests/* +hooks/repo-specific/BiocGenerics_test* From 1491fb420867456b027fa83c79e68636f4ad6271 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Fri, 5 Oct 2018 12:55:20 -0400 Subject: [PATCH 55/70] Add pre-receive-hook-software with changes, - return dictionary to make the hooks more extendable - in the event another software hook which needs to be toggled is added to this, then it can be easily added. --- hooks/repo-specific/pre-receive-hook-software | 107 +++++++++--------- 1 file changed, 53 insertions(+), 54 deletions(-) diff --git a/hooks/repo-specific/pre-receive-hook-software b/hooks/repo-specific/pre-receive-hook-software index cd21673..125afab 100755 --- a/hooks/repo-specific/pre-receive-hook-software +++ b/hooks/repo-specific/pre-receive-hook-software @@ -1,9 +1,7 @@ #!/usr/bin/env python import subprocess -import sys -import os -import re +from os import path, getcwd import fileinput from prevent_large_files import prevent_large_files from prevent_duplicate_commits import prevent_duplicate_commits @@ -13,85 +11,86 @@ ZERO_COMMIT = "0000000000000000000000000000000000000000" HOOKS_CONF = "file:///home/git/repositories/admin/hook_maintainer.git" LOCAL_HOOKS_CONF = "file:////Users/ni41435_ca/Documents/hook_maintainer.git" -# This code doesn't run; enable for local testing. -# to use: -# -# 1. Toggle 'False' to 'True' -# 2. make a git repostiory -# 3. create invalid commit history -# 4. run python prevent-duplicate-commits from repository root -# 5. create branch and repeat for each test -if False: - refname=None - revs = subprocess.check_output([ - "git", "log", "-2", "--format=%H" - ]).splitlines() - newrev = revs[0].strip() - oldrev = revs[1].strip() - prevent_large_files(oldrev, newrev, refname) - prevent_duplicate_commits(oldrev, newrev, refname) - sys.exit(0) - - -# TODO: Change LOCAL_HOOKS_CONF to HOOKS_CONF + def get_hooks_conf(): + """This function does a simple 'git archive' clone process of hooks.conf. + + It clones the file in the /tmp directory. + """ + # FIXME: Change LOCAL_HOOKS_CONF to HOOKS_CONF cmd = "git archive --remote=" + LOCAL_HOOKS_CONF + " HEAD hooks.conf | tar -x" - res = subprocess.check_output(cmd, shell=True, cwd="/tmp") - if os.path.exists("/tmp/hooks.conf"): + subprocess.check_output(cmd, shell=True, cwd="/tmp") + if path.exists("/tmp/hooks.conf"): with open("/tmp/hooks.conf") as f: conf = f.read() return conf -def read_bioc_conf(conf = get_hooks_conf()): +def read_bioc_conf(conf): + """ Read the bioc hooks configuration file. + + This code is run within the 'hooks' folder inside a bare git repo. + + This function reads the hooks.conf file and returns a three tuple + of boolean values, one for each hook if it is toggled False or True. + + Default is (True, True, True) + """ + # Make dictionary with package name as key, values are [list of hooks] d = {} res = [pack.strip().split("\n") for pack in conf.split("\n\n")] for item in res: d[item[0]] = item[1:] - # Get package name, this works because the script is run inside the package - package_name = re.search('/.*/(.+?.git).*', - os.getcwd()).group(1).replace(".git", "") + # Get package name, it works because the script is run inside the package. + package_name = path.basename(path.dirname(getcwd())).replace(".git", "") package = "Package: " + package_name - large_files = True - version_numbers = True - duplicate_commits = True - if "pre-receive-hook-large-files: False" in d[package]: - large_files = False - if "pre-receive-hook-version-numbers: False" in d[package]: - version_numbers = False - if "pre-receive-hook-duplicate-commits: False" in d[package]: - duplicate_commits = False - return (large_files, version_numbers, duplicate_commits) - - -def apply_hooks(large_files, version_numbers, duplicate_commits): + # Default values for hooks is (True, True, True) + hooks_dict = {"pre-receive-hook-large-files": True, + "pre-receive-hook-version-numbers": True, + "pre-receive-hook-duplicate-commits": True} + # Change values for specific hooks mentioned in hooks.conf + for hook in d[package]: + [hook, val] = hook.split(": ") + hooks_dict[hook] = (val != "False") + + return hooks_dict + + +def apply_hooks(hooks_dict): + """Apply hooks to each package. + + This function takes in a boolean list of arguments, one for each hook, + 1. prevent_large_files, + 2. prevent_bad_version_numbers, + 3. prevent_duplicate_commits in that order. + + The boolean values toggle True/False to indicate which hook has to be + applied to the package. + """ newestrev = ZERO_COMMIT oldestrev = ZERO_COMMIT for line in fileinput.input(): std_input = line.split(" ") - print(std_input) oldrev, newrev, refname = [elt.strip() for elt in std_input] # Check for zero commit, check branch deletions if newrev == ZERO_COMMIT: continue # prevent large files - if large_files: # enable hook - print("large_files: ", large_files) + if hooks_dict["pre-receive-hook-large-files"]: # enable hook prevent_large_files(oldrev, newrev, refname) # prevent duplicate commits - if duplicate_commits: # enable hook - print("duplicate commit: ", duplicate_commits) + if hooks_dict["pre-receive-hook-duplicate-commits"]: # enable hook prevent_duplicate_commits(oldrev, newrev, refname) if newestrev == ZERO_COMMIT: newestrev = newrev oldestrev = oldrev - # prevent bad version numbers - print("version_numbers and newestrev != ZERO_COMMIT", version_numbers and newestrev != ZERO_COMMIT) - if (version_numbers and newestrev != ZERO_COMMIT): # enable hook - print("version_numbers: ", version_numbers) + # prevent bad version numbers (enable hook) + if (hooks_dict["pre-receive-hook-version-numbers"] and newestrev != ZERO_COMMIT): prevent_bad_version_numbers(oldestrev, newestrev, refname) + return if __name__ == "__main__": - large_files, version_numbers, duplicate_commits = read_bioc_conf() - apply_hooks(large_files, version_numbers, duplicate_commits) + conf = get_hooks_conf() + hooks_dict = read_bioc_conf(conf) + apply_hooks(hooks_dict) From 723691ed3791c4dd022a61acf1862c2d21ad97e8 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Sun, 7 Oct 2018 18:21:27 -0400 Subject: [PATCH 56/70] follow 80 char rule --- hooks/repo-specific/pre-receive-hook-software | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hooks/repo-specific/pre-receive-hook-software b/hooks/repo-specific/pre-receive-hook-software index 125afab..5d2b97e 100755 --- a/hooks/repo-specific/pre-receive-hook-software +++ b/hooks/repo-specific/pre-receive-hook-software @@ -18,7 +18,8 @@ def get_hooks_conf(): It clones the file in the /tmp directory. """ # FIXME: Change LOCAL_HOOKS_CONF to HOOKS_CONF - cmd = "git archive --remote=" + LOCAL_HOOKS_CONF + " HEAD hooks.conf | tar -x" + cmd = "git archive --remote=" + LOCAL_HOOKS_CONF + \ + " HEAD hooks.conf | tar -x" subprocess.check_output(cmd, shell=True, cwd="/tmp") if path.exists("/tmp/hooks.conf"): with open("/tmp/hooks.conf") as f: @@ -85,7 +86,8 @@ def apply_hooks(hooks_dict): newestrev = newrev oldestrev = oldrev # prevent bad version numbers (enable hook) - if (hooks_dict["pre-receive-hook-version-numbers"] and newestrev != ZERO_COMMIT): + if (hooks_dict["pre-receive-hook-version-numbers"] and + newestrev != ZERO_COMMIT): prevent_bad_version_numbers(oldestrev, newestrev, refname) return From 5804d465190b35b5ae5c28b15fd1e4da516acd0b Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Sun, 7 Oct 2018 18:22:10 -0400 Subject: [PATCH 57/70] Make data-exp and workflow packages modular - this commit allows the hooks.conf to declare data-exp and workflow packages. The only hooks which can go on the data-exp/workflow packages are 1. duplicate commits 2. Version numbers --- .../pre-receive-hook-dataexp-workflow | 76 +++++++++++++++++-- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/hooks/repo-specific/pre-receive-hook-dataexp-workflow b/hooks/repo-specific/pre-receive-hook-dataexp-workflow index 2a06a3b..35c2807 100755 --- a/hooks/repo-specific/pre-receive-hook-dataexp-workflow +++ b/hooks/repo-specific/pre-receive-hook-dataexp-workflow @@ -1,14 +1,71 @@ #!/usr/bin/env python import subprocess -import sys +from os import path, getcwd import fileinput from prevent_duplicate_commits import prevent_duplicate_commits from prevent_bad_version_numbers import prevent_bad_version_numbers ZERO_COMMIT = "0000000000000000000000000000000000000000" +HOOKS_CONF = "file:///home/git/repositories/admin/hook_maintainer.git" +LOCAL_HOOKS_CONF = "file:////Users/ni41435_ca/Documents/hook_maintainer.git" -if __name__ == "__main__": + +def get_hooks_conf(): + """This function does a simple 'git archive' clone process of hooks.conf. + + It clones the file in the /tmp directory. + """ + # FIXME: Change LOCAL_HOOKS_CONF to HOOKS_CONF + cmd = "git archive --remote=" + LOCAL_HOOKS_CONF + \ + " HEAD hooks.conf | tar -x" + subprocess.check_output(cmd, shell=True, cwd="/tmp") + if path.exists("/tmp/hooks.conf"): + with open("/tmp/hooks.conf") as f: + conf = f.read() + return conf + + +def read_bioc_conf(conf): + """ Read the bioc hooks configuration file. + + This code is run within the 'hooks' folder inside a bare git repo. + + This function reads the hooks.conf file and returns a three tuple + of boolean values, one for each hook if it is toggled False or True. + + Default is (True, True, True) + """ + # Make dictionary with package name as key, values are [list of hooks] + d = {} + res = [pack.strip().split("\n") for pack in conf.split("\n\n")] + for item in res: + d[item[0]] = item[1:] + # Get package name, it works because the script is run inside the package. + package_name = path.basename(path.dirname(getcwd())).replace(".git", "") + package = "Package: " + package_name + # Default values for hooks is (True, True) + hooks_dict = {"pre-receive-hook-version-numbers": True, + "pre-receive-hook-duplicate-commits": True} + # Change values for specific hooks mentioned in hooks.conf + for hook in d[package]: + [hook, val] = hook.split(": ") + hooks_dict[hook] = (val != "False") + + return hooks_dict + + +def apply_hooks(hooks_dict): + """Apply hooks to each package in the category data-experiement or + workflow. + + This function takes in a boolean list of arguments, one for each hook, + 1. prevent_bad_version_numbers, + 2. prevent_duplicate_commits in that order. + + The boolean values toggle True/False to indicate which hook has to be + applied to the package. + """ newestrev = ZERO_COMMIT oldestrev = ZERO_COMMIT for line in fileinput.input(): @@ -18,10 +75,19 @@ if __name__ == "__main__": if newrev == ZERO_COMMIT: continue # prevent duplicate commits - prevent_duplicate_commits(oldrev, newrev, refname) + if hooks_dict["pre-receive-hook-duplicate-commits"]: # enable hook + prevent_duplicate_commits(oldrev, newrev, refname) # prevent bad version numbers if newestrev == ZERO_COMMIT: newestrev = newrev oldestrev = oldrev - if newestrev != ZERO_COMMIT: - prevent_bad_version_numbers(oldestrev, newestrev, refname) \ No newline at end of file + if (hooks_dict["pre-receive-hook-version-numbers"] and + newestrev != ZERO_COMMIT): + prevent_bad_version_numbers(oldestrev, newestrev, refname) + return + + +if __name__ == "__main__": + conf = get_hooks_conf() + hooks_dict = read_bioc_conf() + apply_hooks(hooks_dict) From da1d8f3d0d476d3a09b9e0ae5765b42052a2a1be Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Thu, 11 Oct 2018 11:21:16 -0400 Subject: [PATCH 58/70] Add ability to not read comment characters --- .../pre-receive-hook-dataexp-workflow | 11 +++++++++-- hooks/repo-specific/pre-receive-hook-software | 15 +++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/hooks/repo-specific/pre-receive-hook-dataexp-workflow b/hooks/repo-specific/pre-receive-hook-dataexp-workflow index 35c2807..d595547 100755 --- a/hooks/repo-specific/pre-receive-hook-dataexp-workflow +++ b/hooks/repo-specific/pre-receive-hook-dataexp-workflow @@ -12,9 +12,12 @@ LOCAL_HOOKS_CONF = "file:////Users/ni41435_ca/Documents/hook_maintainer.git" def get_hooks_conf(): - """This function does a simple 'git archive' clone process of hooks.conf. + """This function does a simple 'git archive' clone process of + hooks.conf. + + It clones the file in the /tmp directory. This function ignores + the '#' characters in the file. - It clones the file in the /tmp directory. """ # FIXME: Change LOCAL_HOOKS_CONF to HOOKS_CONF cmd = "git archive --remote=" + LOCAL_HOOKS_CONF + \ @@ -23,6 +26,10 @@ def get_hooks_conf(): if path.exists("/tmp/hooks.conf"): with open("/tmp/hooks.conf") as f: conf = f.read() + txt = txt.splitlines() + # Ignore '#' in the file + conf = "\n".join([line for line in txt + if not line.startswith("#")]) return conf diff --git a/hooks/repo-specific/pre-receive-hook-software b/hooks/repo-specific/pre-receive-hook-software index 5d2b97e..5a5299f 100755 --- a/hooks/repo-specific/pre-receive-hook-software +++ b/hooks/repo-specific/pre-receive-hook-software @@ -13,9 +13,12 @@ LOCAL_HOOKS_CONF = "file:////Users/ni41435_ca/Documents/hook_maintainer.git" def get_hooks_conf(): - """This function does a simple 'git archive' clone process of hooks.conf. + """This function does a simple 'git archive' clone process of + hooks.conf. + + It clones the file in the /tmp directory. This function ignores + the '#' characters in the file. - It clones the file in the /tmp directory. """ # FIXME: Change LOCAL_HOOKS_CONF to HOOKS_CONF cmd = "git archive --remote=" + LOCAL_HOOKS_CONF + \ @@ -23,7 +26,11 @@ def get_hooks_conf(): subprocess.check_output(cmd, shell=True, cwd="/tmp") if path.exists("/tmp/hooks.conf"): with open("/tmp/hooks.conf") as f: - conf = f.read() + txt = f.read() + txt = txt.splitlines() + # Ignore '#' in the file + conf = "\n".join([line for line in txt + if not line.startswith("#")]) return conf @@ -39,7 +46,7 @@ def read_bioc_conf(conf): """ # Make dictionary with package name as key, values are [list of hooks] d = {} - res = [pack.strip().split("\n") for pack in conf.split("\n\n")] + res = [pack.strip().split("\n") for pack in conf.split("\n\n") ] for item in res: d[item[0]] = item[1:] # Get package name, it works because the script is run inside the package. From 94e57c977316e1eed4d806f03b55e401f25b4767 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Thu, 11 Oct 2018 14:20:18 -0400 Subject: [PATCH 59/70] Bug fix: Screen for presence of package before making dictionary, this should address To git.bioconductor.org:packages/CRISPRseek.git ! [remote rejected] master -> master (pre-receive hook declined) error: failed to push some refs to 'git@git.bioconductor.org:packages/CRISPRseek.git' and such errors. --- .../repo-specific/pre-receive-hook-dataexp-workflow | 13 ++++++------- hooks/repo-specific/pre-receive-hook-software | 13 ++++++------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/hooks/repo-specific/pre-receive-hook-dataexp-workflow b/hooks/repo-specific/pre-receive-hook-dataexp-workflow index d595547..433cb40 100755 --- a/hooks/repo-specific/pre-receive-hook-dataexp-workflow +++ b/hooks/repo-specific/pre-receive-hook-dataexp-workflow @@ -20,8 +20,7 @@ def get_hooks_conf(): """ # FIXME: Change LOCAL_HOOKS_CONF to HOOKS_CONF - cmd = "git archive --remote=" + LOCAL_HOOKS_CONF + \ - " HEAD hooks.conf | tar -x" + cmd = "git archive --remote=" + HOOKS_CONF + " HEAD hooks.conf | tar -x" subprocess.check_output(cmd, shell=True, cwd="/tmp") if path.exists("/tmp/hooks.conf"): with open("/tmp/hooks.conf") as f: @@ -49,16 +48,16 @@ def read_bioc_conf(conf): for item in res: d[item[0]] = item[1:] # Get package name, it works because the script is run inside the package. - package_name = path.basename(path.dirname(getcwd())).replace(".git", "") + package_name = path.basename(getcwd()).replace(".git", "") package = "Package: " + package_name # Default values for hooks is (True, True) hooks_dict = {"pre-receive-hook-version-numbers": True, "pre-receive-hook-duplicate-commits": True} # Change values for specific hooks mentioned in hooks.conf - for hook in d[package]: - [hook, val] = hook.split(": ") - hooks_dict[hook] = (val != "False") - + if package in d.keys(): + for hook in d[package]: + [hook, val] = hook.split(": ") + hooks_dict[hook] = (val != "False") return hooks_dict diff --git a/hooks/repo-specific/pre-receive-hook-software b/hooks/repo-specific/pre-receive-hook-software index 5a5299f..00d21dd 100755 --- a/hooks/repo-specific/pre-receive-hook-software +++ b/hooks/repo-specific/pre-receive-hook-software @@ -21,8 +21,7 @@ def get_hooks_conf(): """ # FIXME: Change LOCAL_HOOKS_CONF to HOOKS_CONF - cmd = "git archive --remote=" + LOCAL_HOOKS_CONF + \ - " HEAD hooks.conf | tar -x" + cmd = "git archive --remote=" + HOOKS_CONF + " HEAD hooks.conf | tar -x" subprocess.check_output(cmd, shell=True, cwd="/tmp") if path.exists("/tmp/hooks.conf"): with open("/tmp/hooks.conf") as f: @@ -50,17 +49,17 @@ def read_bioc_conf(conf): for item in res: d[item[0]] = item[1:] # Get package name, it works because the script is run inside the package. - package_name = path.basename(path.dirname(getcwd())).replace(".git", "") + package_name = path.basename(getcwd()).replace(".git", "") package = "Package: " + package_name # Default values for hooks is (True, True, True) hooks_dict = {"pre-receive-hook-large-files": True, "pre-receive-hook-version-numbers": True, "pre-receive-hook-duplicate-commits": True} # Change values for specific hooks mentioned in hooks.conf - for hook in d[package]: - [hook, val] = hook.split(": ") - hooks_dict[hook] = (val != "False") - + if package in d.keys(): + for hook in d[package]: + [hook, val] = hook.split(": ") + hooks_dict[hook] = (val != "False") return hooks_dict From b6cd6d19d90f3bdc6afd91915910191b5f0b75dc Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Wed, 24 Oct 2018 16:52:00 -0400 Subject: [PATCH 60/70] Fix bugs in code pointed out by @vobencha This fixes a comment (in master) and how release doesn't check if y and y0 are the same. --- hooks/repo-specific/prevent_bad_version_numbers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hooks/repo-specific/prevent_bad_version_numbers.py b/hooks/repo-specific/prevent_bad_version_numbers.py index 2c48de2..8394a11 100644 --- a/hooks/repo-specific/prevent_bad_version_numbers.py +++ b/hooks/repo-specific/prevent_bad_version_numbers.py @@ -93,11 +93,11 @@ def check_version_in_release(prev_version, new_version): # x should never change if x != x0: throw_error(prev_version, new_version) - # y should be even + # y should be even if y % 2 != 0: - # y should not be 99 i.e no major version change - if y == 99: - throw_error(prev_version, new_version) + throw_error(prev_version, new_version) + # y should not be 99 i.e no major version change + if (y!=y0) or (y == 99): throw_error(prev_version, new_version) # z should be incremented if not z - z0 >= 0: @@ -115,7 +115,7 @@ def check_version_in_master(prev_version, new_version): # y should be odd if y % 2 == 0: throw_error(prev_version, new_version) - # y should not be the same, and can be 99 + # y should be the same, and can be 99 if (y != y0) and (y != 99): throw_error(prev_version, new_version) # z should be incremented and cannot be 99 From 6d7b873dfffa69dfe301a34e780148cbfd52801d Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Thu, 25 Oct 2018 14:47:38 -0400 Subject: [PATCH 61/70] This cleans up the code in the release version --- hooks/repo-specific/prevent_bad_version_numbers.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/hooks/repo-specific/prevent_bad_version_numbers.py b/hooks/repo-specific/prevent_bad_version_numbers.py index 8394a11..8c32214 100644 --- a/hooks/repo-specific/prevent_bad_version_numbers.py +++ b/hooks/repo-specific/prevent_bad_version_numbers.py @@ -90,14 +90,9 @@ def check_version_in_release(prev_version, new_version): """Check version in RELEASE_branch.""" x0, y0, z0 = map(int, prev_version.split(".")) x, y, z = map(int, new_version.split(".")) - # x should never change - if x != x0: - throw_error(prev_version, new_version) - # y should be even - if y % 2 != 0: - throw_error(prev_version, new_version) - # y should not be 99 i.e no major version change - if (y!=y0) or (y == 99): + # x should never change, y should be even, y should not be 99 i.e + # no major version change + if (x != x0) or (y % 2 != 0) or (y!=y0) or (y == 99): throw_error(prev_version, new_version) # z should be incremented if not z - z0 >= 0: From aca8b7a3ed10b3cfc2ebde14371a0f26e0cdc8f6 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Mon, 29 Oct 2018 09:58:05 -0400 Subject: [PATCH 62/70] Adding script to detect bad version numbers --- misc/detect_bad_version.py | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 misc/detect_bad_version.py diff --git a/misc/detect_bad_version.py b/misc/detect_bad_version.py new file mode 100644 index 0000000..49ffd21 --- /dev/null +++ b/misc/detect_bad_version.py @@ -0,0 +1,51 @@ +""" +Usage: + + python detect_bad_version.py + +""" + +import os +import sys + + +def find_description(directory): + description_files = [] + for f in os.listdir(directory): + description_files.append(os.path.join(directory, f, "DESCRIPTION")) + return description_files + + +def check_version(version): + version_number = version.replace("Version :","").split(".") + y = int(version_number[1]) + correct = True + ## Add rules here + if y % 2 == 0: + correct = False + if y > 99: + correct = False + return correct + + +def read_description(DESCRIPTION_path): + with open(DESCRIPTION_path) as f: + txt = f.read() + lines = txt.splitlines() + version = [line for line in lines if line.startswith("Version")][0] + package_name = DESCRIPTION_path.replace("/DESCRIPTION","").replace("packages/","") + return (package_name, version) + + +def run(directory): + descriptions = find_description(directory) + for description in descriptions: + package_name, version = read_description(description) + if not check_version(version): + print(package_name, version) + return + + +if __name__ == "__main__": + print("Directory passed: ", sys.argv[1]) + run(str(sys.argv[1])) From d14d24fc66b37e85850432d8fad4e9848d828df3 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Mon, 29 Oct 2018 09:58:05 -0400 Subject: [PATCH 63/70] Adding script to detect bad version numbers --- misc/detect_bad_version.py | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 misc/detect_bad_version.py diff --git a/misc/detect_bad_version.py b/misc/detect_bad_version.py new file mode 100644 index 0000000..49ffd21 --- /dev/null +++ b/misc/detect_bad_version.py @@ -0,0 +1,51 @@ +""" +Usage: + + python detect_bad_version.py + +""" + +import os +import sys + + +def find_description(directory): + description_files = [] + for f in os.listdir(directory): + description_files.append(os.path.join(directory, f, "DESCRIPTION")) + return description_files + + +def check_version(version): + version_number = version.replace("Version :","").split(".") + y = int(version_number[1]) + correct = True + ## Add rules here + if y % 2 == 0: + correct = False + if y > 99: + correct = False + return correct + + +def read_description(DESCRIPTION_path): + with open(DESCRIPTION_path) as f: + txt = f.read() + lines = txt.splitlines() + version = [line for line in lines if line.startswith("Version")][0] + package_name = DESCRIPTION_path.replace("/DESCRIPTION","").replace("packages/","") + return (package_name, version) + + +def run(directory): + descriptions = find_description(directory) + for description in descriptions: + package_name, version = read_description(description) + if not check_version(version): + print(package_name, version) + return + + +if __name__ == "__main__": + print("Directory passed: ", sys.argv[1]) + run(str(sys.argv[1])) From aafad0456733fc1c9ae89752582763ba56b371f2 Mon Sep 17 00:00:00 2001 From: vobencha Date: Mon, 29 Oct 2018 18:04:58 -0700 Subject: [PATCH 64/70] Add 'parity' argument to detect_bad_version.py --- misc/detect_bad_version.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/misc/detect_bad_version.py b/misc/detect_bad_version.py index 49ffd21..9a71b48 100644 --- a/misc/detect_bad_version.py +++ b/misc/detect_bad_version.py @@ -1,8 +1,10 @@ """ Usage: - python detect_bad_version.py + python detect_bad_version.py + Passing 'even' as the second argument results in a success if the + version is even. Packages with odd versions will be output. """ import os @@ -16,16 +18,21 @@ def find_description(directory): return description_files -def check_version(version): +def check_version(version, parity): version_number = version.replace("Version :","").split(".") - y = int(version_number[1]) - correct = True + y = int(version_number[1]) ## Add rules here - if y % 2 == 0: - correct = False + if parity == "odd": + if y % 2 == 0: + return False + elif parity == "even": + if y % 2 != 0: + return False + if y > 99: - correct = False - return correct + return False + else: + return True def read_description(DESCRIPTION_path): @@ -37,15 +44,15 @@ def read_description(DESCRIPTION_path): return (package_name, version) -def run(directory): +def run(directory, parity): descriptions = find_description(directory) - for description in descriptions: + for description in descriptions: package_name, version = read_description(description) - if not check_version(version): + if not check_version(version, parity): print(package_name, version) return if __name__ == "__main__": - print("Directory passed: ", sys.argv[1]) - run(str(sys.argv[1])) + print("Directory passed: ", sys.argv[1], sys.argv[2]) + run(str(sys.argv[1]), str(sys.argv[2])) From 65e043530c6251046ee991607c7beaaae88cc39b Mon Sep 17 00:00:00 2001 From: vobencha Date: Mon, 29 Oct 2018 18:31:37 -0700 Subject: [PATCH 65/70] Modify detect_bad_version: - Walk directories only - Skip directories without DESCRIPTION --- misc/detect_bad_version.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/misc/detect_bad_version.py b/misc/detect_bad_version.py index 9a71b48..c371dd7 100644 --- a/misc/detect_bad_version.py +++ b/misc/detect_bad_version.py @@ -13,8 +13,11 @@ def find_description(directory): description_files = [] - for f in os.listdir(directory): - description_files.append(os.path.join(directory, f, "DESCRIPTION")) + # Walk directories only; skip those without DESCRIPTION files + for f in os.walk(directory).next()[1]: + dfile = os.path.join(directory, f, "DESCRIPTION") + if os.path.exists(dfile): + description_files.append(dfile) return description_files From 11e6334cd34d1032ffa2d53c33ca051355e53d1e Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Tue, 30 Oct 2018 10:10:42 -0400 Subject: [PATCH 66/70] y == 99 is redundant with y % 2 != 0. This is why the `y==99` condition was removed. It fails anyway. --- hooks/repo-specific/prevent_bad_version_numbers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/repo-specific/prevent_bad_version_numbers.py b/hooks/repo-specific/prevent_bad_version_numbers.py index 8c32214..ccdcf9c 100644 --- a/hooks/repo-specific/prevent_bad_version_numbers.py +++ b/hooks/repo-specific/prevent_bad_version_numbers.py @@ -92,7 +92,7 @@ def check_version_in_release(prev_version, new_version): x, y, z = map(int, new_version.split(".")) # x should never change, y should be even, y should not be 99 i.e # no major version change - if (x != x0) or (y % 2 != 0) or (y!=y0) or (y == 99): + if (x != x0) or (y % 2 != 0) or (y!=y0): throw_error(prev_version, new_version) # z should be incremented if not z - z0 >= 0: From d74fdf3f60cb62a7e8eaacd931ce6d7081de5125 Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Mon, 3 Dec 2018 11:10:50 -0500 Subject: [PATCH 67/70] Prints out the number of packages with bad version. --- misc/detect_bad_version.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/misc/detect_bad_version.py b/misc/detect_bad_version.py index c371dd7..6c0dc3d 100644 --- a/misc/detect_bad_version.py +++ b/misc/detect_bad_version.py @@ -31,7 +31,6 @@ def check_version(version, parity): elif parity == "even": if y % 2 != 0: return False - if y > 99: return False else: @@ -49,13 +48,16 @@ def read_description(DESCRIPTION_path): def run(directory, parity): descriptions = find_description(directory) + counter = [] for description in descriptions: package_name, version = read_description(description) if not check_version(version, parity): print(package_name, version) - return + counter.append(package_name) + return counter if __name__ == "__main__": print("Directory passed: ", sys.argv[1], sys.argv[2]) - run(str(sys.argv[1]), str(sys.argv[2])) + counter = run(str(sys.argv[1]), str(sys.argv[2])) + print(len(counter), " packages have version which is not ", sys.argv[2]) From a916a08a66d2d083c9c452fcfbd828dd5d65da5d Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Fri, 10 May 2019 12:07:02 -0400 Subject: [PATCH 68/70] Update hooks with new pre-receive hook for merge conflicts --- .../pre-receive-hook-dataexp-workflow | 4 +- hooks/repo-specific/pre-receive-hook-software | 16 +++++--- hooks/repo-specific/prevent_merge_markers.py | 40 +++++++++++++++++++ 3 files changed, 53 insertions(+), 7 deletions(-) create mode 100644 hooks/repo-specific/prevent_merge_markers.py diff --git a/hooks/repo-specific/pre-receive-hook-dataexp-workflow b/hooks/repo-specific/pre-receive-hook-dataexp-workflow index 433cb40..6479715 100755 --- a/hooks/repo-specific/pre-receive-hook-dataexp-workflow +++ b/hooks/repo-specific/pre-receive-hook-dataexp-workflow @@ -24,7 +24,7 @@ def get_hooks_conf(): subprocess.check_output(cmd, shell=True, cwd="/tmp") if path.exists("/tmp/hooks.conf"): with open("/tmp/hooks.conf") as f: - conf = f.read() + txt = f.read() txt = txt.splitlines() # Ignore '#' in the file conf = "\n".join([line for line in txt @@ -95,5 +95,5 @@ def apply_hooks(hooks_dict): if __name__ == "__main__": conf = get_hooks_conf() - hooks_dict = read_bioc_conf() + hooks_dict = read_bioc_conf(conf) apply_hooks(hooks_dict) diff --git a/hooks/repo-specific/pre-receive-hook-software b/hooks/repo-specific/pre-receive-hook-software index 00d21dd..60c4994 100755 --- a/hooks/repo-specific/pre-receive-hook-software +++ b/hooks/repo-specific/pre-receive-hook-software @@ -6,6 +6,7 @@ import fileinput from prevent_large_files import prevent_large_files from prevent_duplicate_commits import prevent_duplicate_commits from prevent_bad_version_numbers import prevent_bad_version_numbers +from prevent_merge_markers import prevent_merge_markers ZERO_COMMIT = "0000000000000000000000000000000000000000" HOOKS_CONF = "file:///home/git/repositories/admin/hook_maintainer.git" @@ -41,7 +42,7 @@ def read_bioc_conf(conf): This function reads the hooks.conf file and returns a three tuple of boolean values, one for each hook if it is toggled False or True. - Default is (True, True, True) + Default is (True, True, True, True) """ # Make dictionary with package name as key, values are [list of hooks] d = {} @@ -52,7 +53,8 @@ def read_bioc_conf(conf): package_name = path.basename(getcwd()).replace(".git", "") package = "Package: " + package_name # Default values for hooks is (True, True, True) - hooks_dict = {"pre-receive-hook-large-files": True, + hooks_dict = {"pre-receive-hook-merge-markers": True, + "pre-receive-hook-large-files": True, "pre-receive-hook-version-numbers": True, "pre-receive-hook-duplicate-commits": True} # Change values for specific hooks mentioned in hooks.conf @@ -67,9 +69,10 @@ def apply_hooks(hooks_dict): """Apply hooks to each package. This function takes in a boolean list of arguments, one for each hook, - 1. prevent_large_files, - 2. prevent_bad_version_numbers, - 3. prevent_duplicate_commits in that order. + 1. prevent_merge_markers + 2. prevent_large_files + 3. prevent_bad_version_numbers, + 4. prevent_duplicate_commits in that order. The boolean values toggle True/False to indicate which hook has to be applied to the package. @@ -82,6 +85,9 @@ def apply_hooks(hooks_dict): # Check for zero commit, check branch deletions if newrev == ZERO_COMMIT: continue + # prevent merge conflict markers + if hooks_dict["pre-receive-hook-merge-markers"]: # enable hook + prevent_merge_markers(oldrev, newrev, refname) # prevent large files if hooks_dict["pre-receive-hook-large-files"]: # enable hook prevent_large_files(oldrev, newrev, refname) diff --git a/hooks/repo-specific/prevent_merge_markers.py b/hooks/repo-specific/prevent_merge_markers.py new file mode 100644 index 0000000..4ab4982 --- /dev/null +++ b/hooks/repo-specific/prevent_merge_markers.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +"""Pre-receive hook to check for merge markers in commits. + +This merge marker and merge conflict check pre-receive hook +tries to prevent maintainers from commiting files with <<<, +>>>, === merge markers in them. This keeps the commit history +clean. +""" + +from __future__ import print_function +import subprocess +import sys + + +ZERO_COMMIT = "0000000000000000000000000000000000000000" + + +def git_diff_files_with_conflicts(oldrev, newrev): + """Get list of files in diff.""" + files_modified = subprocess.check_output(['git', + 'diff', + '--name-only', + '-G"<<<<<|=====|>>>>>"', + oldrev + ".." + newrev]) + return files_modified.splitlines() + + +def prevent_merge_markers(oldrev, newrev, refname): + """Prevent merge markers in files. + + This function prevents merge markers in commits. + """ + conflicts = git_diff_files_with_conflicts(oldrev, newrev) + # If number of files with conflicts is > 0 + if conflicts: + message = ("Error: You cannot commit without resolving merge conflicts.\n" + "Unresolved merge conlicts in these files: \n" + + ", ".join(conflicts)) + sys.exit(message) + return From 9ca29f9e8058b755163e12bf9324ec1063d0182d Mon Sep 17 00:00:00 2001 From: Nitesh Turaga Date: Mon, 16 Sep 2019 09:47:20 -0400 Subject: [PATCH 69/70] update bad version numbers code and test --- .../prevent_bad_version_numbers.py | 2 +- .../test_prevent_bad_version_numbers.py | 34 ++++++++++++------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/hooks/repo-specific/prevent_bad_version_numbers.py b/hooks/repo-specific/prevent_bad_version_numbers.py index ccdcf9c..082af5e 100644 --- a/hooks/repo-specific/prevent_bad_version_numbers.py +++ b/hooks/repo-specific/prevent_bad_version_numbers.py @@ -74,7 +74,7 @@ def get_version_bump(diff): def check_version_format(prev_version, new_version): """Check format of version.""" - regex = re.compile('\d{1}\.\d{1,2}\.\d{1,3}$') + regex = re.compile(r'\d+\.\d+\.\d+$') if not regex.match(new_version): throw_error(prev_version, new_version) try: diff --git a/hooks/repo-specific/test_prevent_bad_version_numbers.py b/hooks/repo-specific/test_prevent_bad_version_numbers.py index 3ef28db..debb097 100644 --- a/hooks/repo-specific/test_prevent_bad_version_numbers.py +++ b/hooks/repo-specific/test_prevent_bad_version_numbers.py @@ -4,7 +4,7 @@ import re import os import pytest -from hooks.prevent_bad_version_numbers import check_version_bump +from prevent_bad_version_numbers import check_version_bump CWD = "/Users/ni41435_ca/Documents/bioc_git_transition/hooks/repo-specific/test_proj" DESC = "DESCRIPTION" @@ -233,14 +233,24 @@ def test_devel_version_bumps_martin(): return -# def test_integration_version_bump_in_master(): -# refname = "master" -# change_version("1.25.61") -# git_add(DESC) -# with pytest.raises(SystemExit) as pytest_wrapped_e: -# change_version("2.25.61") -# git_add(DESC) -# git_commit("Fail test", cwd=CWD) -# out = git_push() -# assert "error" in out -# assert pytest_wrapped_e.value.code == 1 +def test_devel_version_bumps_sep2019(): + refname = "master" + + res = check_version_bump("2.3.2", "2.3.999", refname) + assert res == 0 + + res = check_version_bump("1.7.999", "1.7.1000", refname) + assert res == 0 + + res = check_version_bump("1.7.999", "1.7.9991", refname) + assert res == 0 + + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("1.7.999", "1.7.10", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code != 0 + + with pytest.raises(SystemExit) as pytest_wrapped_e: + check_version_bump("a1.7.999", "a1.7.1000", refname) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code != 0 From d2d4304bd08add7ec9cc7d82bc4ff9a203a28b89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Pag=C3=A8s?= Date: Tue, 1 Nov 2022 17:48:30 -0400 Subject: [PATCH 70/70] Migrate detect_duplicate_commits.py from Python 2 to Python 3 --- misc/detect_duplicate_commits.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/misc/detect_duplicate_commits.py b/misc/detect_duplicate_commits.py index 3cf2b06..a5f9acb 100644 --- a/misc/detect_duplicate_commits.py +++ b/misc/detect_duplicate_commits.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import subprocess import sys @@ -32,14 +32,23 @@ to see body of commits. """ +def bytes2str(line): + if isinstance(line, str): + return line + try: + line = line.decode() # decode() uses utf-8 encoding by default + except UnicodeDecodeError: + line = line.decode("iso8859") # typical Windows encoding + return line + def get_svn_revision(commit): body = subprocess.check_output([ "git", "show", "--format=%b", commit ]) + body = bytes2str(body) revision = SVN_COMMIT_REGEX.match(body) if revision != None: revision = revision.group(1) return revision - def prevent_duplicate_commits(newrev): """Pre-receive hook to check for duplicate SVN commits.""" try: @@ -49,11 +58,12 @@ def prevent_duplicate_commits(newrev): except Exception as e: print("Exception: %s" % e) pass + commit_list = bytes2str(commit_list) commit_list = commit_list.split("\n") commit_list = [item for item in commit_list if len(item)>0] # For each of the first GIT_COMMIT_LIST_LENGTH pairs, check diff - for i in xrange(len(commit_list) - 1): + for i in range(len(commit_list) - 1): first = commit_list[i] second = commit_list[i+1]