diff --git a/LFS_README.md b/LFS_README.md new file mode 100644 index 0000000..9a6c80e --- /dev/null +++ b/LFS_README.md @@ -0,0 +1,15 @@ +# LFS branch + +This branch is a backup of the original code in LFS.py which versions the experiment data using git-lfs. + +Files needed: + +1. src/LFS.py + +2. run_transition.py + +## Problems + +1. git-lfs server set up + +2. gitolite diff --git a/src/git_api/git_api.py b/src/git_api/git_api.py index 5614cc4..495fc11 100644 --- a/src/git_api/git_api.py +++ b/src/git_api/git_api.py @@ -115,8 +115,8 @@ def git_add(path, cwd): return -def git_commit(cwd): +def git_commit(message, cwd): """Commit files to git server.""" - cmd = ['git', 'commit', '-m', 'Adding external data files as LFS'] + cmd = ['git', 'commit', '-m', message] subprocess.check_call(cmd, cwd=cwd) return diff --git a/src/git_edit_repository.py b/src/git_edit_repository.py deleted file mode 100644 index 47f7d08..0000000 --- a/src/git_edit_repository.py +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/env python - -"""Bioconductor Git repo user interaction script docstrings. - -This module provides functions for working with the Bioconductor -`git` repository. This module gives the bioconductor core team, -to interact with the GIT server, and work with package authors. - -Author: Nitesh Turaga - -# TODO: This needs to be edited to fit new constraints. -""" -import os -import subprocess -from src.git_api.git_api import git_clone -from src.git_api.git_api import git_remote_add -from src.git_api.git_api import git_checkout -from local_svn_dump import Singleton -import logging as log - - -class GitEditRepository(object): - """Git Edit repository.""" - __metaclass__ = Singleton - - def __init__(self, edit_repo, ssh_server): - """Initialize Git edit repo.""" - self.edit_repo = edit_repo - self.ssh_server = ssh_server - return - - # TODO: deprecate - def extract_development_url(self, package): - """Extract `DevelopmentURL` from DESCRIPTION file.""" - description = os.path.join(package, 'DESCRIPTION') - with open(description, 'r') as f: - doc = f.read() - doc_list = doc.split("\n") - for i in xrange(len(doc_list)): - if doc_list[i].startswith("DevelopmentURL:"): - url = doc_list[i] - url = url.replace("DevelopmentURL:", "") - url = url.strip() - return url - - # TODO: deprecate - def set_edit_repo(self, package): - """ - Clone a package from bioc_git_repo to make changes. - - Use this function to set up a clone of an existing bioconductor-git - package, to make changes and push back to the git-server. - """ - log.info("Set up a clone of package: %s, " - "to push changes to bare_git_repo" % package) - repository = self.ssh_server + "/" + package - git_clone(repository, self.edit_repo, bare=False) - development_url = self.extract_development_url(os.path.join(self.edit_repo, package)) - git_remote_add('upstream', development_url, - cwd=os.path.join(self.edit_repo, package)) - return - - # TODO: deprecate - def clone_all_edit_repo(self): - """Clone all packages in git server. - - This clone of the entire git server is located on a seperate instance, - where contents of the pacakges may be edited. After the modifications - in the `edit_repo`, the contents can be pushed to the `bioc_git_repo`. - """ - # TODO: get list of package from servers - # get list of packages from manifest file. - for package in os.listdir(self.bare_git_repo): - self.set_edit_repo(package) - return - - # TODO: deprecate - def daily_fetch_branch(self, package, branch): - """Daily fetch from github repo. - - ## git checkout branch-1 - ## git pull upstream branch-1:branch-1 - ## git push origin branch-1 - """ - path = os.path.join(self.edit_repo, package) - git_checkout(branch, cwd=path, new=False) - # Git pull into a branch of the edit_repp - cmd = ['git', 'pull', '-Xtheirs', '--no-edit', 'upstream', - branch + ":" + branch] - p = subprocess.Popen(cmd, cwd=path, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out, err = p.communicate() - if ("CONFLICT" in out): - log.error("Merge conflict in the package: %s" % package) - log.error(out) - else: # Git push - cmd = ['git', 'push', 'origin', branch] - subprocess.check_call(cmd, cwd=path) - return - - # TODO: deprecate - def daily_fetch(self, branch): - """Daily fetch of every package in the edit_repo. - - Daily fetch needs to be run by the build system on - the branch being updated e.g `master` or `RELEASE_3_4` - """ - for package in os.listdir(self.edit_repo): - self.daily_fetch_branch(package, branch) - return - - def version_bump(self, package, release=False): - """Bump package version. - - release=False, assumes that version bump is not for a new - release branch. - """ - description_file = os.path.join(self.edit_repo, package, 'DESCRIPTION') - with open(description_file, 'r') as f: - doc = f.read() - doc_list = doc.split("\n") - for i in xrange(len(doc_list)): - if doc_list[i].startswith("Version:"): - version = doc_list[i] - index = i - x, y, z = version.split("Version: ")[1].split(".") - if release: - # Special case - if int(y) == 99: - x = int(x) + 1 - y = 0 - else: - y = int(y) + 1 - z = 0 - else: - z = int(z) + 1 - version = str(x) + "." + str(y) + "." + str(z) - doc_list[index] = "Version: " + version - with open(description_file, "w") as f: - f.write("\n".join(doc_list)) - log.info("Package: %s updated version to: %s" % (package, version)) - return - - # TODO: REMOVE commit_message and use the one in git_api - def commit_message(self, msg, package): - """Add a commit message to package during, bioc release.""" - cmd = ['git', 'commit', '-m', msg] - subprocess.check_call(cmd, cwd=package) - return - - def release_branch(self, package, new_release): - """Create new release branch, make git call.""" - package_dir = os.path.join(self.edit_repo, package) - # checkout master - git_checkout(branch="master", cwd=package_dir, new=False) - # on master, version bump, release = False - self.version_bump(package, release=False) - self.commit_message("bump x.y.z versions to even 'y' " - "prior to creation of " + new_release, package_dir) - - # IN THE BRANCH, version bump release=True - git_checkout(branch=new_release, cwd=package_dir, new=True) - self.version_bump(package, release=True) - # commit messsage in branch - self.commit_message(msg="Creating branch for BioC " + new_release, - package=package_dir) - # checkout master - git_checkout(branch="master", cwd=package_dir, new=False) - # version bump release=True - self.version_bump(package, release=True) - # Commit message - self.commit_message("bump x.y.z versions to odd 'y' " - "after creation of " + new_release, package_dir) - log.info("New branch created for package %s" % package) - return - - def create_new_release_branch(self, new_release): - """Create a new RELEASE from master. - - This function is used to create a new release version, - from the 'master' branch going forward. - Usage: create_new_release_branch('RELEASE_3_7', '/packages/') - """ - for package in os.listdir(os.path.abspath(self.edit_repo)): - # Create a new release branch - self.release_branch(new_release, package) - # TODO: Push new release branch - cmd = ['git', 'push', '-u', 'origin', new_release] - subprocess.check_call(cmd, cwd=os.path.join(self.edit_repo, package)) - return diff --git a/src/lfs.py b/src/lfs.py index 9d5b81c..529ee9b 100644 --- a/src/lfs.py +++ b/src/lfs.py @@ -96,7 +96,8 @@ def commit_data_to_lfs(self, package): """Commit data as LFS to server.""" try: package_dir = os.path.join(self.temp_git_repo, package) - git_commit(cwd=package_dir) + git_commit(message="Adding external data files as LFS", + cwd=package_dir) except Exception as e: log.error("Error commit data to LFS in package %s" % package) log.error(e) diff --git a/src/local_svn_dump.py b/src/local_svn_dump.py index 0a3aac6..fbb7e95 100644 --- a/src/local_svn_dump.py +++ b/src/local_svn_dump.py @@ -18,7 +18,8 @@ class Singleton(type): def __call__(cls, *args, **kwargs): if cls not in cls._instances: - cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) + cls._instances[cls] = super(Singleton, cls).__call__(*args, + **kwargs) return cls._instances[cls] @@ -74,6 +75,18 @@ def manifest_package_list(self, manifest_file): for line in doc if line.startswith("Package")] return package_list + def search_git_files(self, path): + """Check if path has pre exisiting .git files.""" + cmd = 'svn list --depth=infinity ' + path + " | grep \\.git" + try: + proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output, error = proc.communicate() + except Exception as e: + log.error("Error in search git files: %s" % path) + log.error(e) + return output + def svn_dump(self, packs): """ Create git svn clone from SVN dump for each package. @@ -84,17 +97,18 @@ def svn_dump(self, packs): package_dir = self.svn_root + '/' + 'trunk' + self.package_path for pack in packs: package_dump = os.path.join(package_dir, pack) - # TODO: git svn clone from each release branch. - # This will be tricky. - try: - cmd = ['git', 'svn', 'clone', - '--authors-file=' + self.users_db, package_dump] - subprocess.check_call(cmd, cwd=self.bioc_git_repo) - log.debug("Finished git-svn clone for package: %s" % pack) - except subprocess.CalledProcessError as e: - log.error("Error : %s in package %s" % (e, pack)) - except Exception as e: # All other errors - log.error("Unexpected error: %s" % e) + # If .git files exsit in package, throw error. + pre_exisiting_git = self.search_git_files(package_dump) + if not pre_exisiting_git: + try: + cmd = ['git', 'svn', 'clone', + '--authors-file=' + self.users_db, package_dump] + subprocess.check_call(cmd, cwd=self.bioc_git_repo) + log.debug("Finished git-svn clone for package: %s" % pack) + except subprocess.CalledProcessError as e: + log.error("Error : %s in package %s" % (e, pack)) + except Exception as e: # All other errors + log.error("Unexpected error: %s" % e) return def svn_get_revision(self): diff --git a/src/release_process.py b/src/release_process.py new file mode 100644 index 0000000..f16bda0 --- /dev/null +++ b/src/release_process.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python + +"""Bioconductor Git repo user interaction script docstrings. + +This module provides functions for working with the Bioconductor +`git` repository. This module gives the bioconductor core team, +to interact with the git server during the release process. + +Author: Nitesh Turaga +""" +import os +import subprocess +from src.git_api.git_api import git_commit +from src.git_api.git_api import git_checkout +from local_svn_dump import Singleton +import logging as log + + +class ReleaseProcess(object): + """Git Edit repository.""" + __metaclass__ = Singleton + + def __init__(self, bare_git_repo): + """Initialize Git edit repo.""" + self.bare_git_repo = bare_git_repo + return + + def version_bump(self, package, release=False): + """Bump package version. + + release=False, assumes that version bump is not for a new + release branch. + """ + description_file = os.path.join(self.bare_git_repo, package, + 'DESCRIPTION') + with open(description_file, 'r') as f: + doc = f.read() + doc_list = doc.split("\n") + for i in xrange(len(doc_list)): + if doc_list[i].startswith("Version:"): + version = doc_list[i] + index = i + x, y, z = version.split("Version: ")[1].split(".") + if release: + # Special case + if int(y) == 99: + x = int(x) + 1 + y = 0 + else: + y = int(y) + 1 + z = 0 + else: + z = int(z) + 1 + version = str(x) + "." + str(y) + "." + str(z) + doc_list[index] = "Version: " + version + with open(description_file, "w") as f: + f.write("\n".join(doc_list)) + log.info("Package: %s updated version to: %s" % (package, version)) + return + + def release_branch(self, package, new_release): + """Create new release branch, make git call.""" + package_dir = os.path.join(self.bare_git_repo, package) + # checkout master + git_checkout(branch="master", cwd=package_dir, new=False) + # on master, version bump, release = False + self.version_bump(package, release=False) + msg1 = ("bump x.y.z versions to even 'y' prior to creation of " + + new_release) + git_commit(msg1, cwd=package_dir) + # IN THE BRANCH, version bump release=True + git_checkout(branch=new_release, cwd=package_dir, new=True) + self.version_bump(package, release=True) + # commit messsage in branch + git_commit("Creating branch for BioC " + new_release, + cwd=package_dir) + # checkout master + git_checkout(branch="master", cwd=package_dir, new=False) + # version bump release=True + self.version_bump(package, release=True) + # Commit message + msg2 = ("bump x.y.z versions to odd 'y' after creation of " + + new_release) + git_commit(msg2, cwd=package_dir) + log.info("New branch created for package %s" % package) + return + + def create_new_release_branch(self, new_release): + """Create a new RELEASE from master. + + This function is used to create a new release version, + from the 'master' branch going forward. + Usage: create_new_release_branch('RELEASE_3_7', '/packages/') + """ + for package in os.listdir(os.path.abspath(self.bare_git_repo)): + # Create a new release branch + self.release_branch(new_release, package) + # TODO: Push new release branch + cmd = ['git', 'push', '-u', 'origin', new_release] + subprocess.check_call(cmd, cwd=os.path.join(self.bare_git_repo, + package)) + return